mirror of
https://github.com/Dvorinka/Devour.git
synced 2026-06-03 20:13:03 +00:00
first commit
This commit is contained in:
Tomas Dvorak
@@ -0,0 +1,266 @@
|
||||
package nuxtdocs
|
||||
|
||||
import (
|
||||
"net/url"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/PuerkitoBio/goquery"
|
||||
)
|
||||
|
||||
type Parser struct {
|
||||
baseURL string
|
||||
}
|
||||
|
||||
func NewParser() *Parser {
|
||||
return &Parser{
|
||||
baseURL: "https://nuxt.com",
|
||||
}
|
||||
}
|
||||
|
||||
func (p *Parser) ParseReferencePage(html string, docURL string) (*Reference, error) {
|
||||
doc, err := goquery.NewDocumentFromReader(strings.NewReader(html))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
ref := &Reference{
|
||||
DocURL: docURL,
|
||||
FetchedAt: time.Now(),
|
||||
}
|
||||
|
||||
ref.Components = p.extractComponents(doc, docURL)
|
||||
ref.Composables = p.extractComposables(doc, docURL)
|
||||
ref.Utilities = p.extractUtilities(doc, docURL)
|
||||
ref.Configs = p.extractConfigs(doc, docURL)
|
||||
ref.Commands = p.extractCommands(doc, docURL)
|
||||
|
||||
return ref, nil
|
||||
}
|
||||
|
||||
func (p *Parser) ParseSearchResults(html string) ([]*SearchResult, error) {
|
||||
doc, err := goquery.NewDocumentFromReader(strings.NewReader(html))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var results []*SearchResult
|
||||
|
||||
doc.Find(".search-result, a[href*='/docs/'], a[href*='/api/'], .nav-link").Each(func(i int, s *goquery.Selection) {
|
||||
result := &SearchResult{}
|
||||
|
||||
result.Name = strings.TrimSpace(s.Text())
|
||||
|
||||
if href, exists := s.Attr("href"); exists {
|
||||
result.DocURL = resolveURL(p.baseURL, href)
|
||||
|
||||
if strings.Contains(href, "/components/") {
|
||||
result.Kind = "component"
|
||||
} else if strings.Contains(href, "/composables/") {
|
||||
result.Kind = "composable"
|
||||
} else if strings.Contains(href, "/utils/") {
|
||||
result.Kind = "utility"
|
||||
} else if strings.Contains(href, "/config/") || strings.Contains(href, "/configuration/") {
|
||||
result.Kind = "config"
|
||||
} else if strings.Contains(href, "/commands/") {
|
||||
result.Kind = "command"
|
||||
} else {
|
||||
result.Kind = "doc"
|
||||
}
|
||||
}
|
||||
|
||||
if result.Name != "" {
|
||||
results = append(results, result)
|
||||
}
|
||||
})
|
||||
|
||||
return results, nil
|
||||
}
|
||||
|
||||
func (p *Parser) extractComponents(doc *goquery.Document, docURL string) []*Component {
|
||||
var components []*Component
|
||||
|
||||
nuxtComponents := []string{"NuxtPage", "NuxtLayout", "NuxtLink", "NuxtLoadingIndicator", "NuxtErrorBoundary", "NuxtPicture", "NuxtImg", "ClientOnly", "DevOnly"}
|
||||
|
||||
doc.Find("h1, h2, h3, .api-item, [id]").Each(func(_ int, s *goquery.Selection) {
|
||||
id, _ := s.Attr("id")
|
||||
text := s.Text()
|
||||
|
||||
for _, name := range nuxtComponents {
|
||||
if id == name || strings.Contains(text, name) || strings.Contains(text, "<"+name) {
|
||||
comp := &Component{
|
||||
Name: name,
|
||||
}
|
||||
|
||||
comp.DocURL = docURL + "#" + name
|
||||
|
||||
next := s.Next()
|
||||
for next.Length() > 0 && !next.Is("h1, h2, h3") {
|
||||
if next.Is("p") && comp.Doc == "" {
|
||||
comp.Doc = strings.TrimSpace(next.Text())
|
||||
}
|
||||
next = next.Next()
|
||||
}
|
||||
|
||||
components = append(components, comp)
|
||||
break
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
return components
|
||||
}
|
||||
|
||||
func (p *Parser) extractComposables(doc *goquery.Document, docURL string) []*Composable {
|
||||
var composables []*Composable
|
||||
|
||||
nuxtComposables := []string{"useAsyncData", "useFetch", "useLazyAsyncData", "useLazyFetch", "useNuxtData", "useHead", "useHeadSafe", "useSeoMeta", "useRoute", "useRouter", "useState", "useCookie", "useRequestURL", "useRequestEvent", "useRequestHeaders", "useResponseHeader", "useRuntimeConfig", "useAppConfig", "useError", "createError", "isNuxtError", "showError", "throwError", "clearError", "reloadNuxtApp", "useRequestFetch", "useHydration", "usePreviewMode", "onPrehydrate"}
|
||||
|
||||
doc.Find("h1, h2, h3, .api-item, code").Each(func(_ int, s *goquery.Selection) {
|
||||
id, _ := s.Attr("id")
|
||||
text := s.Text()
|
||||
|
||||
for _, name := range nuxtComposables {
|
||||
if id == name || strings.Contains(text, name+"(") {
|
||||
comp := &Composable{
|
||||
Name: name,
|
||||
}
|
||||
|
||||
comp.DocURL = docURL + "#" + name
|
||||
|
||||
next := s.Next()
|
||||
for next.Length() > 0 && !next.Is("h1, h2, h3") {
|
||||
if next.Is("p") && comp.Doc == "" {
|
||||
comp.Doc = strings.TrimSpace(next.Text())
|
||||
}
|
||||
next = next.Next()
|
||||
}
|
||||
|
||||
composables = append(composables, comp)
|
||||
break
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
return composables
|
||||
}
|
||||
|
||||
func (p *Parser) extractUtilities(doc *goquery.Document, docURL string) []*Utility {
|
||||
var utilities []*Utility
|
||||
|
||||
nuxtUtils := []string{"navigateTo", "abortNavigation", "setPageLayout", "defineNuxtComponent", "defineNuxtPlugin", "definePayloadHandler", "defineNuxtRouteMiddleware", "definePageMeta", "defineNuxtModule", "addComponent", "addImports", "addPluginTemplate", "createResolver"}
|
||||
|
||||
doc.Find("h1, h2, h3, .api-item, code").Each(func(_ int, s *goquery.Selection) {
|
||||
id, _ := s.Attr("id")
|
||||
text := s.Text()
|
||||
|
||||
for _, name := range nuxtUtils {
|
||||
if id == name || strings.Contains(text, name+"(") {
|
||||
util := &Utility{
|
||||
Name: name,
|
||||
}
|
||||
|
||||
util.DocURL = docURL + "#" + name
|
||||
|
||||
next := s.Next()
|
||||
for next.Length() > 0 && !next.Is("h1, h2, h3") {
|
||||
if next.Is("p") && util.Doc == "" {
|
||||
util.Doc = strings.TrimSpace(next.Text())
|
||||
}
|
||||
next = next.Next()
|
||||
}
|
||||
|
||||
utilities = append(utilities, util)
|
||||
break
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
return utilities
|
||||
}
|
||||
|
||||
func (p *Parser) extractConfigs(doc *goquery.Document, docURL string) []*Config {
|
||||
var configs []*Config
|
||||
|
||||
nuxtConfigs := []string{"app", "build", "builder", "components", "compatibilityDate", "content", "css", "devtools", "extends", "experimental", "features", "generate", "hooks", "ignore", "imports", "logLevel", "modules", "nitro", "optimization", "pages", "plugins", "postcss", "prepare", "rootDir", "runtimeConfig", "serverDir", "sourcemap", "srcDir", "ssr", "telemetry", "testUtils", "typescript", "vite", "vue", "watchers", "workspaceDir"}
|
||||
|
||||
doc.Find("h1, h2, h3, .api-item, [id]").Each(func(_ int, s *goquery.Selection) {
|
||||
id, _ := s.Attr("id")
|
||||
text := s.Text()
|
||||
|
||||
for _, name := range nuxtConfigs {
|
||||
if id == name || strings.Contains(strings.ToLower(text), name) {
|
||||
cfg := &Config{
|
||||
Name: name,
|
||||
}
|
||||
|
||||
cfg.DocURL = docURL + "#" + name
|
||||
|
||||
next := s.Next()
|
||||
for next.Length() > 0 && !next.Is("h1, h2, h3") {
|
||||
if next.Is("p") && cfg.Doc == "" {
|
||||
cfg.Doc = strings.TrimSpace(next.Text())
|
||||
}
|
||||
next = next.Next()
|
||||
}
|
||||
|
||||
configs = append(configs, cfg)
|
||||
break
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
return configs
|
||||
}
|
||||
|
||||
func (p *Parser) extractCommands(doc *goquery.Document, docURL string) []*Command {
|
||||
var commands []*Command
|
||||
|
||||
nuxtCommands := []string{"nuxi dev", "nuxi build", "nuxi generate", "nuxi preview", "nuxi analyze", "nuxi cleanup", "nuxi typecheck", "nuxi module", "nuxi info", "nuxi prepare", "nuxi upgrade"}
|
||||
|
||||
doc.Find("h1, h2, h3, .api-item, code, pre").Each(func(_ int, s *goquery.Selection) {
|
||||
id, _ := s.Attr("id")
|
||||
text := s.Text()
|
||||
|
||||
for _, name := range nuxtCommands {
|
||||
if id == name || strings.Contains(text, name) {
|
||||
cmd := &Command{
|
||||
Name: name,
|
||||
}
|
||||
|
||||
cmd.DocURL = docURL + "#" + strings.ReplaceAll(name, " ", "-")
|
||||
|
||||
next := s.Next()
|
||||
for next.Length() > 0 && !next.Is("h1, h2, h3") {
|
||||
if next.Is("p") && cmd.Doc == "" {
|
||||
cmd.Doc = strings.TrimSpace(next.Text())
|
||||
}
|
||||
next = next.Next()
|
||||
}
|
||||
|
||||
commands = append(commands, cmd)
|
||||
break
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
return commands
|
||||
}
|
||||
|
||||
func resolveURL(base string, href string) string {
|
||||
if strings.HasPrefix(href, "http") {
|
||||
return href
|
||||
}
|
||||
|
||||
baseURL, err := url.Parse(base)
|
||||
if err != nil {
|
||||
return href
|
||||
}
|
||||
|
||||
hrefURL, err := url.Parse(href)
|
||||
if err != nil {
|
||||
return href
|
||||
}
|
||||
|
||||
return baseURL.ResolveReference(hrefURL).String()
|
||||
}
|
||||
@@ -0,0 +1,126 @@
|
||||
package nuxtdocs
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/PuerkitoBio/goquery"
|
||||
)
|
||||
|
||||
const testReferencePageHTML = `
|
||||
<!DOCTYPE html>
|
||||
<html>
|
||||
<body>
|
||||
<h1>Nuxt API Reference</h1>
|
||||
|
||||
<h2 id="useFetch">useFetch</h2>
|
||||
<p>Fetches data from an API endpoint with SSR support.</p>
|
||||
<pre><code>const { data, pending, error } = await useFetch('/api/data')</code></pre>
|
||||
|
||||
<h2 id="useState">useState</h2>
|
||||
<p>Creates a reactive state that is shared across components.</p>
|
||||
|
||||
<h2 id="NuxtPage">NuxtPage</h2>
|
||||
<p>Renders the current page component based on the route.</p>
|
||||
|
||||
<h2 id="server">server</h2>
|
||||
<p>Nuxt server configuration options.</p>
|
||||
|
||||
<h3 id="nuxi-dev">nuxi dev</h3>
|
||||
<p>Starts the development server.</p>
|
||||
</body>
|
||||
</html>
|
||||
`
|
||||
|
||||
func TestParseReferencePage(t *testing.T) {
|
||||
parser := NewParser()
|
||||
ref, err := parser.ParseReferencePage(testReferencePageHTML, "https://nuxt.com/docs/api/")
|
||||
if err != nil {
|
||||
t.Fatalf("ParseReferencePage failed: %v", err)
|
||||
}
|
||||
|
||||
if len(ref.Composables) == 0 {
|
||||
t.Error("Expected at least one composable")
|
||||
}
|
||||
|
||||
if len(ref.Components) == 0 {
|
||||
t.Error("Expected at least one component")
|
||||
}
|
||||
|
||||
if len(ref.Commands) == 0 {
|
||||
t.Error("Expected at least one command")
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractComposables(t *testing.T) {
|
||||
parser := NewParser()
|
||||
|
||||
doc, err := goquery.NewDocumentFromReader(strings.NewReader(testReferencePageHTML))
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to parse HTML: %v", err)
|
||||
}
|
||||
|
||||
composables := parser.extractComposables(doc, "https://nuxt.com/docs/api/")
|
||||
|
||||
if len(composables) == 0 {
|
||||
t.Fatal("Expected at least one composable")
|
||||
}
|
||||
|
||||
found := false
|
||||
for _, c := range composables {
|
||||
if c.Name == "useFetch" {
|
||||
found = true
|
||||
if c.Doc == "" {
|
||||
t.Error("Expected useFetch to have documentation")
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if !found {
|
||||
t.Error("Expected to find useFetch composable")
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractComponents(t *testing.T) {
|
||||
parser := NewParser()
|
||||
|
||||
doc, err := goquery.NewDocumentFromReader(strings.NewReader(testReferencePageHTML))
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to parse HTML: %v", err)
|
||||
}
|
||||
|
||||
components := parser.extractComponents(doc, "https://nuxt.com/docs/api/")
|
||||
|
||||
found := false
|
||||
for _, c := range components {
|
||||
if c.Name == "NuxtPage" {
|
||||
found = true
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if !found {
|
||||
t.Error("Expected to find NuxtPage component")
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveURL(t *testing.T) {
|
||||
tests := []struct {
|
||||
base string
|
||||
href string
|
||||
expected string
|
||||
}{
|
||||
{"https://nuxt.com", "/docs/api/", "https://nuxt.com/docs/api/"},
|
||||
{"https://nuxt.com", "https://example.com/page", "https://example.com/page"},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.href, func(t *testing.T) {
|
||||
got := resolveURL(tt.base, tt.href)
|
||||
if got != tt.expected {
|
||||
t.Errorf("resolveURL(%q, %q) = %q, want %q", tt.base, tt.href, got, tt.expected)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,159 @@
|
||||
// Package nuxtdocs provides parsing and extraction for Nuxt documentation
|
||||
// from nuxt.com/docs/.
|
||||
package nuxtdocs
|
||||
|
||||
import "time"
|
||||
|
||||
// Reference represents the Nuxt API reference.
|
||||
type Reference struct {
|
||||
Version string `json:"version"`
|
||||
DocURL string `json:"doc_url"`
|
||||
Components []*Component `json:"components,omitempty"`
|
||||
Composables []*Composable `json:"composables,omitempty"`
|
||||
Utilities []*Utility `json:"utilities,omitempty"`
|
||||
Configs []*Config `json:"configs,omitempty"`
|
||||
Commands []*Command `json:"commands,omitempty"`
|
||||
FetchedAt time.Time `json:"fetched_at"`
|
||||
}
|
||||
|
||||
// Component represents a Nuxt component.
|
||||
type Component struct {
|
||||
Name string `json:"name"`
|
||||
Doc string `json:"doc,omitempty"`
|
||||
Props []*Prop `json:"props,omitempty"`
|
||||
Slots []*Slot `json:"slots,omitempty"`
|
||||
Events []*Event `json:"events,omitempty"`
|
||||
Examples []*Example `json:"examples,omitempty"`
|
||||
Category string `json:"category,omitempty"`
|
||||
DocURL string `json:"doc_url"`
|
||||
Deprecated string `json:"deprecated,omitempty"`
|
||||
}
|
||||
|
||||
// Prop represents a component prop.
|
||||
type Prop struct {
|
||||
Name string `json:"name"`
|
||||
Type string `json:"type,omitempty"`
|
||||
Doc string `json:"doc,omitempty"`
|
||||
Required bool `json:"required"`
|
||||
Default string `json:"default,omitempty"`
|
||||
Deprecated string `json:"deprecated,omitempty"`
|
||||
}
|
||||
|
||||
// Slot represents a component slot.
|
||||
type Slot struct {
|
||||
Name string `json:"name"`
|
||||
Doc string `json:"doc,omitempty"`
|
||||
Props string `json:"props,omitempty"`
|
||||
}
|
||||
|
||||
// Event represents a component event.
|
||||
type Event struct {
|
||||
Name string `json:"name"`
|
||||
Doc string `json:"doc,omitempty"`
|
||||
Parameters []*Parameter `json:"parameters,omitempty"`
|
||||
}
|
||||
|
||||
// Composable represents a Nuxt composable.
|
||||
type Composable struct {
|
||||
Name string `json:"name"`
|
||||
Doc string `json:"doc,omitempty"`
|
||||
Signature string `json:"signature,omitempty"`
|
||||
Parameters []*Parameter `json:"parameters,omitempty"`
|
||||
Returns string `json:"returns,omitempty"`
|
||||
Examples []*Example `json:"examples,omitempty"`
|
||||
Category string `json:"category,omitempty"` // data, state, routing, etc.
|
||||
DocURL string `json:"doc_url"`
|
||||
Deprecated string `json:"deprecated,omitempty"`
|
||||
}
|
||||
|
||||
// Utility represents a Nuxt utility function.
|
||||
type Utility struct {
|
||||
Name string `json:"name"`
|
||||
Doc string `json:"doc,omitempty"`
|
||||
Signature string `json:"signature,omitempty"`
|
||||
Parameters []*Parameter `json:"parameters,omitempty"`
|
||||
Returns string `json:"returns,omitempty"`
|
||||
Examples []*Example `json:"examples,omitempty"`
|
||||
DocURL string `json:"doc_url"`
|
||||
Deprecated string `json:"deprecated,omitempty"`
|
||||
}
|
||||
|
||||
// Config represents a Nuxt configuration option.
|
||||
type Config struct {
|
||||
Name string `json:"name"`
|
||||
Doc string `json:"doc,omitempty"`
|
||||
Type string `json:"type,omitempty"`
|
||||
Default string `json:"default,omitempty"`
|
||||
Properties []*Property `json:"properties,omitempty"`
|
||||
Examples []*Example `json:"examples,omitempty"`
|
||||
Category string `json:"category,omitempty"`
|
||||
DocURL string `json:"doc_url"`
|
||||
Deprecated string `json:"deprecated,omitempty"`
|
||||
}
|
||||
|
||||
// Property represents a nested config property.
|
||||
type Property struct {
|
||||
Name string `json:"name"`
|
||||
Doc string `json:"doc,omitempty"`
|
||||
Type string `json:"type,omitempty"`
|
||||
Default string `json:"default,omitempty"`
|
||||
Properties []*Property `json:"properties,omitempty"`
|
||||
Deprecated string `json:"deprecated,omitempty"`
|
||||
}
|
||||
|
||||
// Command represents a Nuxt CLI command.
|
||||
type Command struct {
|
||||
Name string `json:"name"`
|
||||
Doc string `json:"doc,omitempty"`
|
||||
Usage string `json:"usage,omitempty"`
|
||||
Flags []*Flag `json:"flags,omitempty"`
|
||||
Arguments []*Argument `json:"arguments,omitempty"`
|
||||
Examples []*Example `json:"examples,omitempty"`
|
||||
DocURL string `json:"doc_url"`
|
||||
}
|
||||
|
||||
// Flag represents a command flag.
|
||||
type Flag struct {
|
||||
Name string `json:"name"`
|
||||
Doc string `json:"doc,omitempty"`
|
||||
Type string `json:"type,omitempty"`
|
||||
Default string `json:"default,omitempty"`
|
||||
Required bool `json:"required"`
|
||||
Short string `json:"short,omitempty"`
|
||||
}
|
||||
|
||||
// Argument represents a command argument.
|
||||
type Argument struct {
|
||||
Name string `json:"name"`
|
||||
Doc string `json:"doc,omitempty"`
|
||||
Type string `json:"type,omitempty"`
|
||||
Required bool `json:"required"`
|
||||
Default string `json:"default,omitempty"`
|
||||
}
|
||||
|
||||
// Parameter represents a function parameter.
|
||||
type Parameter struct {
|
||||
Name string `json:"name"`
|
||||
Type string `json:"type,omitempty"`
|
||||
Doc string `json:"doc,omitempty"`
|
||||
Optional bool `json:"optional"`
|
||||
Default string `json:"default,omitempty"`
|
||||
}
|
||||
|
||||
// Example represents a code example.
|
||||
type Example struct {
|
||||
Title string `json:"title,omitempty"`
|
||||
Code string `json:"code"`
|
||||
Description string `json:"description,omitempty"`
|
||||
Language string `json:"language,omitempty"`
|
||||
}
|
||||
|
||||
// SearchResult represents a search result.
|
||||
type SearchResult struct {
|
||||
Name string `json:"name"`
|
||||
Kind string `json:"kind"` // component, composable, utility, config, command
|
||||
Doc string `json:"doc,omitempty"`
|
||||
DocURL string `json:"doc_url"`
|
||||
Score int `json:"score"`
|
||||
Deprecated bool `json:"deprecated"`
|
||||
}
|
||||
Reference in New Issue
Block a user