first commit

pkg/astrodocs/parser.go

@@ -0,0 +1,177 @@
+package astrodocs
+
+import (
+	"net/url"
+	"strings"
+	"time"
+
+	"github.com/PuerkitoBio/goquery"
+)
+
+type Parser struct {
+	baseURL string
+}
+
+func NewParser() *Parser {
+	return &Parser{
+		baseURL: "https://docs.astro.build",
+	}
+}
+
+func (p *Parser) ParsePage(html string, docURL string) (*Page, error) {
+	doc, err := goquery.NewDocumentFromReader(strings.NewReader(html))
+	if err != nil {
+		return nil, err
+	}
+
+	page := &Page{
+		URL:       docURL,
+		FetchedAt: time.Now(),
+	}
+
+	page.Title = p.extractTitle(doc)
+	page.Description = p.extractDescription(doc)
+	page.Content = p.extractContent(doc)
+	page.Sections = p.extractSections(doc, docURL)
+	page.CodeBlocks = p.extractCodeBlocks(doc)
+
+	return page, nil
+}
+
+func (p *Parser) ParseSidebar(html string) ([]*Section, error) {
+	doc, err := goquery.NewDocumentFromReader(strings.NewReader(html))
+	if err != nil {
+		return nil, err
+	}
+
+	var sections []*Section
+
+	doc.Find(".sidebar a, nav a, [data-sidebar] a").Each(func(_ int, s *goquery.Selection) {
+		section := &Section{}
+
+		section.Title = strings.TrimSpace(s.Text())
+
+		if href, exists := s.Attr("href"); exists {
+			section.DocURL = resolveURL(p.baseURL, href)
+		}
+
+		if section.Title != "" && section.DocURL != "" {
+			sections = append(sections, section)
+		}
+	})
+
+	return sections, nil
+}
+
+func (p *Parser) extractTitle(doc *goquery.Document) string {
+	title := doc.Find("h1").First().Text()
+	title = strings.TrimSpace(title)
+
+	if title == "" {
+		title = doc.Find("title").First().Text()
+		title = strings.TrimSpace(title)
+		if idx := strings.Index(title, " | "); idx > 0 {
+			title = title[:idx]
+		}
+	}
+
+	return title
+}
+
+func (p *Parser) extractDescription(doc *goquery.Document) string {
+	desc := doc.Find("meta[name='description']").AttrOr("content", "")
+	if desc != "" {
+		return desc
+	}
+
+	return doc.Find("meta[property='og:description']").AttrOr("content", "")
+}
+
+func (p *Parser) extractContent(doc *goquery.Document) string {
+	content := doc.Find("article, main, .content, .sl-markdown-content").First()
+	if content.Length() == 0 {
+		content = doc.Find("body")
+	}
+
+	return strings.TrimSpace(content.Text())
+}
+
+func (p *Parser) extractSections(doc *goquery.Document, docURL string) []*Section {
+	var sections []*Section
+
+	doc.Find("h1, h2, h3").Each(func(_ int, s *goquery.Selection) {
+		section := &Section{}
+
+		section.Title = strings.TrimSpace(s.Text())
+
+		if id, exists := s.Attr("id"); exists {
+			section.ID = id
+			section.DocURL = docURL + "#" + id
+		} else {
+			section.DocURL = docURL
+		}
+
+		if section.Title != "" {
+			sections = append(sections, section)
+		}
+	})
+
+	return sections
+}
+
+func (p *Parser) extractCodeBlocks(doc *goquery.Document) []*CodeBlock {
+	var blocks []*CodeBlock
+
+	doc.Find("pre code, pre").Each(func(_ int, s *goquery.Selection) {
+		block := &CodeBlock{}
+
+		if classes, exists := s.Attr("class"); exists {
+			parts := strings.Split(classes, " ")
+			for _, part := range parts {
+				if strings.HasPrefix(part, "language-") {
+					block.Language = strings.TrimPrefix(part, "language-")
+					break
+				}
+			}
+		}
+
+		if block.Language == "" {
+			parent := s.Parent()
+			if classes, exists := parent.Attr("class"); exists {
+				parts := strings.Split(classes, " ")
+				for _, part := range parts {
+					if strings.HasPrefix(part, "language-") {
+						block.Language = strings.TrimPrefix(part, "language-")
+						break
+					}
+				}
+			}
+		}
+
+		block.Code = strings.TrimSpace(s.Text())
+
+		if block.Code != "" {
+			blocks = append(blocks, block)
+		}
+	})
+
+	return blocks
+}
+
+func resolveURL(base string, href string) string {
+	if strings.HasPrefix(href, "http") {
+		return href
+	}
+
+	baseURL, err := url.Parse(base)
+	if err != nil {
+		return href
+	}
+
+	hrefURL, err := url.Parse(href)
+	if err != nil {
+		return href
+	}
+
+	return baseURL.ResolveReference(hrefURL).String()
+}

pkg/astrodocs/types.go

@@ -0,0 +1,32 @@
+package astrodocs
+
+import "time"
+
+type Page struct {
+	Title       string       `json:"title"`
+	Description string       `json:"description,omitempty"`
+	Content     string       `json:"content,omitempty"`
+	URL         string       `json:"url"`
+	Sections    []*Section   `json:"sections,omitempty"`
+	CodeBlocks  []*CodeBlock `json:"code_blocks,omitempty"`
+	Guides      []*Guide     `json:"guides,omitempty"`
+	FetchedAt   time.Time    `json:"fetched_at"`
+}
+
+type Section struct {
+	ID      string `json:"id"`
+	Title   string `json:"title"`
+	Content string `json:"content,omitempty"`
+	DocURL  string `json:"doc_url"`
+}
+
+type CodeBlock struct {
+	Language string `json:"language,omitempty"`
+	Code     string `json:"code"`
+}
+
+type Guide struct {
+	Title       string `json:"title"`
+	Description string `json:"description,omitempty"`
+	DocURL      string `json:"doc_url"`
+}

pkg/client/client.go

@@ -0,0 +1,61 @@
+// Package client provides a Go client for Devour server.
+package client
+
+import (
+	"context"
+	"net/http"
+)
+
+// Client is a Devour API client.
+type Client struct {
+	baseURL    string
+	httpClient *http.Client
+}
+
+// New creates a new Devour client.
+func New(baseURL string) *Client {
+	return &Client{
+		baseURL:    baseURL,
+		httpClient: &http.Client{},
+	}
+}
+
+// Query searches indexed documents.
+func (c *Client) Query(ctx context.Context, query string, limit int) (*QueryResponse, error) {
+	// TODO: Implement HTTP request to /query
+	return nil, nil
+}
+
+// AddDocuments adds documents to the index.
+func (c *Client) AddDocuments(ctx context.Context, docs []*Document) error {
+	// TODO: Implement HTTP request to /documents
+	return nil
+}
+
+// Status returns index status.
+func (c *Client) Status(ctx context.Context) (*Status, error) {
+	// TODO: Implement HTTP request to /status
+	return nil, nil
+}
+
+// QueryResponse represents a query response.
+type QueryResponse struct {
+	Query   string        `json:"query"`
+	Results []interface{} `json:"results"`
+	Total   int           `json:"total"`
+	TookMs  int64         `json:"took_ms"`
+}
+
+// Document represents a document.
+type Document struct {
+	ID       string                 `json:"id"`
+	Content  string                 `json:"content"`
+	Metadata map[string]interface{} `json:"metadata,omitempty"`
+}
+
+// Status represents index status.
+type Status struct {
+	Healthy       bool `json:"healthy"`
+	DocumentCount int  `json:"document_count"`
+	ChunkCount    int  `json:"chunk_count"`
+}

pkg/cloudflaredocs/parser.go

@@ -0,0 +1,207 @@
+package cloudflaredocs
+
+import (
+	"net/url"
+	"strings"
+	"time"
+
+	"github.com/PuerkitoBio/goquery"
+)
+
+type Parser struct {
+	baseURL string
+}
+
+func NewParser() *Parser {
+	return &Parser{
+		baseURL: "https://developers.cloudflare.com",
+	}
+}
+
+func (p *Parser) ParsePage(html string, docURL string) (*Page, error) {
+	doc, err := goquery.NewDocumentFromReader(strings.NewReader(html))
+	if err != nil {
+		return nil, err
+	}
+
+	page := &Page{
+		URL:       docURL,
+		FetchedAt: time.Now(),
+	}
+
+	page.Title = p.extractTitle(doc)
+	page.Description = p.extractDescription(doc)
+	page.Product = p.extractProduct(doc)
+	page.Content = p.extractContent(doc)
+	page.Sections = p.extractSections(doc, docURL)
+	page.CodeBlocks = p.extractCodeBlocks(doc)
+	page.APIs = p.extractAPIs(doc, docURL)
+
+	return page, nil
+}
+
+func (p *Parser) ParseSidebar(html string) ([]*Section, error) {
+	doc, err := goquery.NewDocumentFromReader(strings.NewReader(html))
+	if err != nil {
+		return nil, err
+	}
+
+	var sections []*Section
+
+	doc.Find(".sidebar a, nav a, [data-sidebar] a, .sl-sidebar a").Each(func(_ int, s *goquery.Selection) {
+		section := &Section{}
+
+		section.Title = strings.TrimSpace(s.Text())
+
+		if href, exists := s.Attr("href"); exists {
+			section.DocURL = resolveURL(p.baseURL, href)
+		}
+
+		if section.Title != "" && section.DocURL != "" {
+			sections = append(sections, section)
+		}
+	})
+
+	return sections, nil
+}
+
+func (p *Parser) extractTitle(doc *goquery.Document) string {
+	title := doc.Find("h1").First().Text()
+	title = strings.TrimSpace(title)
+
+	if title == "" {
+		title = doc.Find("title").First().Text()
+		title = strings.TrimSpace(title)
+		title = strings.TrimSuffix(title, " | Cloudflare Docs")
+		title = strings.TrimSpace(title)
+	}
+
+	return title
+}
+
+func (p *Parser) extractDescription(doc *goquery.Document) string {
+	desc := doc.Find("meta[name='description']").AttrOr("content", "")
+	if desc != "" {
+		return desc
+	}
+
+	desc = doc.Find("meta[property='og:description']").AttrOr("content", "")
+	if desc != "" {
+		return desc
+	}
+
+	return ""
+}
+
+func (p *Parser) extractProduct(doc *goquery.Document) string {
+	product := doc.Find(".product-name, [data-product], .breadcrumb a:first-child").First().Text()
+	return strings.TrimSpace(product)
+}
+
+func (p *Parser) extractContent(doc *goquery.Document) string {
+	content := doc.Find("article, main, .content, .sl-markdown-content").First()
+	if content.Length() == 0 {
+		content = doc.Find("body")
+	}
+
+	return strings.TrimSpace(content.Text())
+}
+
+func (p *Parser) extractSections(doc *goquery.Document, docURL string) []*Section {
+	var sections []*Section
+
+	doc.Find("h1, h2, h3").Each(func(_ int, s *goquery.Selection) {
+		section := &Section{}
+
+		section.Title = strings.TrimSpace(s.Text())
+
+		if id, exists := s.Attr("id"); exists {
+			section.ID = id
+			section.DocURL = docURL + "#" + id
+		} else {
+			section.DocURL = docURL
+		}
+
+		if section.Title != "" {
+			sections = append(sections, section)
+		}
+	})
+
+	return sections
+}
+
+func (p *Parser) extractCodeBlocks(doc *goquery.Document) []*CodeBlock {
+	var blocks []*CodeBlock
+
+	doc.Find("pre code, pre").Each(func(_ int, s *goquery.Selection) {
+		block := &CodeBlock{}
+
+		if classes, exists := s.Attr("class"); exists {
+			parts := strings.Split(classes, " ")
+			for _, part := range parts {
+				if strings.HasPrefix(part, "language-") {
+					block.Language = strings.TrimPrefix(part, "language-")
+					break
+				}
+			}
+		}
+
+		block.Code = strings.TrimSpace(s.Text())
+
+		if block.Code != "" {
+			blocks = append(blocks, block)
+		}
+	})
+
+	return blocks
+}
+
+func (p *Parser) extractAPIs(doc *goquery.Document, docURL string) []*API {
+	var apis []*API
+
+	apiMethods := []string{"GET", "POST", "PUT", "DELETE", "PATCH"}
+
+	doc.Find("pre code, code, .api-endpoint").Each(func(_ int, s *goquery.Selection) {
+		text := strings.TrimSpace(s.Text())
+
+		for _, method := range apiMethods {
+			if strings.HasPrefix(text, method+" ") || strings.HasPrefix(text, method+"\t") {
+				api := &API{
+					Method: method,
+					DocURL: docURL,
+				}
+
+				parts := strings.Fields(text)
+				if len(parts) >= 2 {
+					api.Endpoint = parts[1]
+					api.Name = parts[1]
+				}
+
+				if api.Endpoint != "" {
+					apis = append(apis, api)
+				}
+				break
+			}
+		}
+	})
+
+	return apis
+}
+
+func resolveURL(base string, href string) string {
+	if strings.HasPrefix(href, "http") {
+		return href
+	}
+
+	baseURL, err := url.Parse(base)
+	if err != nil {
+		return href
+	}
+
+	hrefURL, err := url.Parse(href)
+	if err != nil {
+		return href
+	}
+
+	return baseURL.ResolveReference(hrefURL).String()
+}

pkg/cloudflaredocs/types.go

@@ -0,0 +1,35 @@
+package cloudflaredocs
+
+import "time"
+
+type Page struct {
+	Title       string       `json:"title"`
+	Description string       `json:"description,omitempty"`
+	Content     string       `json:"content,omitempty"`
+	URL         string       `json:"url"`
+	Product     string       `json:"product,omitempty"`
+	Sections    []*Section   `json:"sections,omitempty"`
+	CodeBlocks  []*CodeBlock `json:"code_blocks,omitempty"`
+	APIs        []*API       `json:"apis,omitempty"`
+	FetchedAt   time.Time    `json:"fetched_at"`
+}
+
+type Section struct {
+	ID      string `json:"id"`
+	Title   string `json:"title"`
+	Content string `json:"content,omitempty"`
+	DocURL  string `json:"doc_url"`
+}
+
+type CodeBlock struct {
+	Language string `json:"language,omitempty"`
+	Code     string `json:"code"`
+}
+
+type API struct {
+	Name        string `json:"name"`
+	Method      string `json:"method,omitempty"`
+	Endpoint    string `json:"endpoint,omitempty"`
+	Description string `json:"description,omitempty"`
+	DocURL      string `json:"doc_url"`
+}

pkg/dockerdocs/parser.go

@@ -0,0 +1,200 @@
+package dockerdocs
+
+import (
+	"net/url"
+	"strings"
+	"time"
+
+	"github.com/PuerkitoBio/goquery"
+)
+
+type Parser struct {
+	baseURL string
+}
+
+func NewParser() *Parser {
+	return &Parser{
+		baseURL: "https://docs.docker.com",
+	}
+}
+
+func (p *Parser) ParsePage(html string, docURL string) (*Page, error) {
+	doc, err := goquery.NewDocumentFromReader(strings.NewReader(html))
+	if err != nil {
+		return nil, err
+	}
+
+	page := &Page{
+		URL:       docURL,
+		FetchedAt: time.Now(),
+	}
+
+	page.Title = p.extractTitle(doc)
+	page.Description = p.extractDescription(doc)
+	page.Content = p.extractContent(doc)
+	page.Sections = p.extractSections(doc, docURL)
+	page.CodeBlocks = p.extractCodeBlocks(doc)
+	page.Links = p.extractLinks(doc, docURL)
+
+	return page, nil
+}
+
+func (p *Parser) ParseToc(html string) ([]*Section, error) {
+	doc, err := goquery.NewDocumentFromReader(strings.NewReader(html))
+	if err != nil {
+		return nil, err
+	}
+
+	var sections []*Section
+
+	doc.Find("nav a, .toc a, .sidebar a, [data-toc] a").Each(func(_ int, s *goquery.Selection) {
+		section := &Section{}
+
+		section.Title = strings.TrimSpace(s.Text())
+
+		if href, exists := s.Attr("href"); exists {
+			section.DocURL = resolveURL(p.baseURL, href)
+		}
+
+		if section.Title != "" {
+			sections = append(sections, section)
+		}
+	})
+
+	return sections, nil
+}
+
+func (p *Parser) extractTitle(doc *goquery.Document) string {
+	title := doc.Find("h1").First().Text()
+	title = strings.TrimSpace(title)
+
+	if title == "" {
+		title = doc.Find("title").First().Text()
+		title = strings.TrimSpace(title)
+		if idx := strings.Index(title, " | "); idx > 0 {
+			title = title[:idx]
+		}
+		if idx := strings.Index(title, " - "); idx > 0 {
+			title = title[:idx]
+		}
+	}
+
+	return title
+}
+
+func (p *Parser) extractDescription(doc *goquery.Document) string {
+	desc := doc.Find("meta[name='description']").AttrOr("content", "")
+	if desc != "" {
+		return desc
+	}
+
+	desc = doc.Find(".lead, .intro, .summary, p:first-of-type").First().Text()
+	return strings.TrimSpace(desc)
+}
+
+func (p *Parser) extractContent(doc *goquery.Document) string {
+	content := doc.Find("article, main, .content, .documentation, .doc-content").First()
+	if content.Length() == 0 {
+		content = doc.Find("body")
+	}
+
+	return strings.TrimSpace(content.Text())
+}
+
+func (p *Parser) extractSections(doc *goquery.Document, docURL string) []*Section {
+	var sections []*Section
+
+	doc.Find("h1, h2, h3").Each(func(_ int, s *goquery.Selection) {
+		section := &Section{}
+
+		section.Title = strings.TrimSpace(s.Text())
+
+		if id, exists := s.Attr("id"); exists {
+			section.ID = id
+			section.DocURL = docURL + "#" + id
+		} else {
+			section.DocURL = docURL
+		}
+
+		next := s.Next()
+		var content strings.Builder
+		for next.Length() > 0 && !next.Is("h1, h2, h3") {
+			content.WriteString(next.Text())
+			content.WriteString("\n")
+			next = next.Next()
+		}
+		section.Content = strings.TrimSpace(content.String())
+
+		if section.Title != "" {
+			sections = append(sections, section)
+		}
+	})
+
+	return sections
+}
+
+func (p *Parser) extractCodeBlocks(doc *goquery.Document) []*CodeBlock {
+	var blocks []*CodeBlock
+
+	doc.Find("pre code, pre").Each(func(_ int, s *goquery.Selection) {
+		block := &CodeBlock{}
+
+		if classes, exists := s.Attr("class"); exists {
+			if strings.Contains(classes, "language-") {
+				parts := strings.Split(classes, " ")
+				for _, part := range parts {
+					if strings.HasPrefix(part, "language-") {
+						block.Language = strings.TrimPrefix(part, "language-")
+						break
+					}
+				}
+			}
+		}
+
+		block.Code = strings.TrimSpace(s.Text())
+
+		if block.Code != "" {
+			blocks = append(blocks, block)
+		}
+	})
+
+	return blocks
+}
+
+func (p *Parser) extractLinks(doc *goquery.Document, baseURL string) []string {
+	var links []string
+	seen := make(map[string]bool)
+
+	doc.Find("a[href]").Each(func(_ int, s *goquery.Selection) {
+		href, _ := s.Attr("href")
+		if href == "" || strings.HasPrefix(href, "#") || strings.HasPrefix(href, "javascript:") {
+			return
+		}
+
+		resolved := resolveURL(baseURL, href)
+		if !seen[resolved] {
+			seen[resolved] = true
+			links = append(links, resolved)
+		}
+	})
+
+	return links
+}
+
+func resolveURL(base string, href string) string {
+	if strings.HasPrefix(href, "http") {
+		return href
+	}
+
+	baseURL, err := url.Parse(base)
+	if err != nil {
+		return href
+	}
+
+	hrefURL, err := url.Parse(href)
+	if err != nil {
+		return href
+	}
+
+	return baseURL.ResolveReference(hrefURL).String()
+}

pkg/dockerdocs/types.go

@@ -0,0 +1,26 @@
+package dockerdocs
+
+import "time"
+
+type Page struct {
+	Title       string       `json:"title"`
+	Description string       `json:"description,omitempty"`
+	Content     string       `json:"content,omitempty"`
+	URL         string       `json:"url"`
+	Sections    []*Section   `json:"sections,omitempty"`
+	CodeBlocks  []*CodeBlock `json:"code_blocks,omitempty"`
+	Links       []string     `json:"links,omitempty"`
+	FetchedAt   time.Time    `json:"fetched_at"`
+}
+
+type Section struct {
+	ID      string `json:"id"`
+	Title   string `json:"title"`
+	Content string `json:"content,omitempty"`
+	DocURL  string `json:"doc_url"`
+}
+
+type CodeBlock struct {
+	Language string `json:"language,omitempty"`
+	Code     string `json:"code"`
+}

pkg/godocs/parser.go

@@ -0,0 +1,687 @@
+package godocs
+
+import (
+	"net/url"
+	"regexp"
+	"strings"
+	"time"
+
+	"github.com/PuerkitoBio/goquery"
+)
+
+// Parser parses pkg.go.dev HTML pages into structured documentation.
+type Parser struct {
+	baseURL string
+}
+
+// NewParser creates a new parser for pkg.go.dev content.
+func NewParser() *Parser {
+	return &Parser{
+		baseURL: "https://pkg.go.dev",
+	}
+}
+
+// ParsePackagePage parses a pkg.go.dev package documentation page.
+func (p *Parser) ParsePackagePage(html string, docURL string) (*Package, error) {
+	doc, err := goquery.NewDocumentFromReader(strings.NewReader(html))
+	if err != nil {
+		return nil, err
+	}
+
+	pkg := &Package{
+		DocURL:    docURL,
+		FetchedAt: time.Now(),
+	}
+
+	// Extract import path from URL or breadcrumb
+	pkg.ImportPath = p.extractImportPath(doc, docURL)
+	pkg.Name = p.extractPackageName(doc)
+
+	// Extract synopsis
+	pkg.Synopsis = p.extractSynopsis(doc)
+
+	// Extract package documentation
+	pkg.Doc = p.extractPackageDoc(doc)
+
+	// Extract version info
+	pkg.Version = p.extractVersion(doc)
+
+	// Extract module info
+	pkg.Module = p.extractModule(doc)
+
+	// Extract licenses
+	pkg.Licenses = p.extractLicenses(doc)
+
+	// Extract imported by count
+	pkg.ImportedBy = p.extractImportedBy(doc)
+
+	// Extract repository URL
+	pkg.Repository = p.extractRepository(doc)
+
+	// Extract functions
+	pkg.Functions = p.extractFunctions(doc)
+
+	// Extract types
+	pkg.Types = p.extractTypes(doc)
+
+	// Extract constants
+	pkg.Constants = p.extractConstants(doc)
+
+	// Extract variables
+	pkg.Variables = p.extractVariables(doc)
+
+	// Extract examples
+	pkg.Examples = p.extractPackageExamples(doc)
+
+	return pkg, nil
+}
+
+// ParseSearchResults parses pkg.go.dev search results page.
+func (p *Parser) ParseSearchResults(html string) ([]*SearchResult, error) {
+	doc, err := goquery.NewDocumentFromReader(strings.NewReader(html))
+	if err != nil {
+		return nil, err
+	}
+
+	var results []*SearchResult
+
+	doc.Find(".SearchSnippet").Each(func(i int, s *goquery.Selection) {
+		result := &SearchResult{}
+
+		// Extract name and path
+		s.Find("h2 a").Each(func(_ int, a *goquery.Selection) {
+			result.Name = strings.TrimSpace(a.Text())
+			if href, exists := a.Attr("href"); exists {
+				result.URL = p.baseURL + href
+				result.Path = strings.TrimPrefix(href, "/")
+			}
+		})
+
+		// Extract path from span
+		pathSpan := s.Find(".SearchSnippet-header-path")
+		if pathSpan.Length() > 0 {
+			result.Path = strings.Trim(pathSpan.Text(), "()")
+		}
+
+		// Extract synopsis
+		synopsis := s.Find(".SearchSnippet-synopsis")
+		if synopsis.Length() > 0 {
+			result.Synopsis = strings.TrimSpace(synopsis.Text())
+		}
+
+		// Extract imported by count
+		infoLabel := s.Find(".SearchSnippet-infoLabel").Text()
+		if strings.Contains(infoLabel, "Imported by") {
+			re := regexp.MustCompile(`Imported by\s+(\d[\d,]*)`)
+			if matches := re.FindStringSubmatch(infoLabel); len(matches) > 1 {
+				countStr := strings.ReplaceAll(matches[1], ",", "")
+				result.ImportedBy = parseCount(countStr)
+			}
+		}
+
+		// Extract version
+		versionMatch := regexp.MustCompile(`v?\d+\.\d+(?:\.\d+)?`).FindString(infoLabel)
+		result.Version = versionMatch
+
+		// Extract license
+		license := s.Find("[data-test-id='snippet-license'] a")
+		if license.Length() > 0 {
+			result.License = strings.TrimSpace(license.Text())
+		}
+
+		results = append(results, result)
+	})
+
+	return results, nil
+}
+
+// extractImportPath extracts the import path from the page.
+func (p *Parser) extractImportPath(doc *goquery.Document, docURL string) string {
+	// Try to extract from breadcrumb
+	var importPath string
+	doc.Find(".go-Breadcrumb li a").Each(func(i int, s *goquery.Selection) {
+		if i > 0 { // Skip "Discover Packages"
+			part := strings.TrimSpace(s.Text())
+			if part != "" {
+				if importPath != "" {
+					importPath += "/"
+				}
+				importPath += part
+			}
+		}
+	})
+
+	if importPath != "" {
+		return importPath
+	}
+
+	// Fallback: extract from URL
+	if docURL != "" {
+		u, err := url.Parse(docURL)
+		if err == nil {
+			path := strings.TrimPrefix(u.Path, "/")
+			// Remove version suffix like @v1.0.0
+			if idx := strings.Index(path, "@"); idx > 0 {
+				path = path[:idx]
+			}
+			return path
+		}
+	}
+
+	return ""
+}
+
+// extractPackageName extracts the package name.
+func (p *Parser) extractPackageName(doc *goquery.Document) string {
+	// Try UnitHeader-title
+	title := doc.Find(".UnitHeader-titleHeading").Text()
+	title = strings.TrimSpace(title)
+	if title != "" {
+		return title
+	}
+
+	// Fallback to h1
+	title = doc.Find("h1").First().Text()
+	return strings.TrimSpace(title)
+}
+
+// extractSynopsis extracts the package synopsis.
+func (p *Parser) extractSynopsis(doc *goquery.Document) string {
+	// Synopsis is typically in the first paragraph after the package declaration
+	docSection := doc.Find(".Documentation").First()
+	if docSection.Length() > 0 {
+		// Get the first paragraph
+		firstP := docSection.Find("p").First()
+		if firstP.Length() > 0 {
+			synopsis := strings.TrimSpace(firstP.Text())
+			// Limit to reasonable length
+			if len(synopsis) > 200 {
+				synopsis = synopsis[:197] + "..."
+			}
+			return synopsis
+		}
+	}
+
+	return ""
+}
+
+// extractPackageDoc extracts the full package documentation.
+func (p *Parser) extractPackageDoc(doc *goquery.Document) string {
+	var parts []string
+
+	doc.Find(".Documentation").Each(func(_ int, s *goquery.Selection) {
+		text := s.Text()
+		text = cleanWhitespace(text)
+		if text != "" {
+			parts = append(parts, text)
+		}
+	})
+
+	return strings.Join(parts, "\n\n")
+}
+
+// extractVersion extracts the version info.
+func (p *Parser) extractVersion(doc *goquery.Document) string {
+	versionEl := doc.Find("[data-test-id='UnitHeader-version'] a")
+	if versionEl.Length() > 0 {
+		return strings.TrimSpace(versionEl.Text())
+	}
+	return ""
+}
+
+// extractModule extracts module information.
+func (p *Parser) extractModule(doc *goquery.Document) *Module {
+	modulePath := ""
+	moduleVersion := ""
+
+	// Try to extract from version link
+	versionEl := doc.Find("[data-test-id='UnitHeader-version'] a")
+	if versionEl.Length() > 0 {
+		moduleVersion = strings.TrimSpace(versionEl.Text())
+	}
+
+	// Extract module path from breadcrumb
+	doc.Find(".go-Breadcrumb li a").Each(func(i int, s *goquery.Selection) {
+		text := strings.TrimSpace(s.Text())
+		if strings.Contains(text, "/") && i > 0 {
+			modulePath = text
+		}
+	})
+
+	if modulePath != "" {
+		return &Module{
+			Path:    modulePath,
+			Version: moduleVersion,
+		}
+	}
+
+	return nil
+}
+
+// extractLicenses extracts license information.
+func (p *Parser) extractLicenses(doc *goquery.Document) []License {
+	var licenses []License
+
+	doc.Find("[data-test-id='UnitHeader-license']").Each(func(_ int, s *goquery.Selection) {
+		name := strings.TrimSpace(s.Text())
+		if name != "" {
+			license := License{Name: name}
+			if href, exists := s.Attr("href"); exists {
+				license.Path = href
+			}
+			licenses = append(licenses, license)
+		}
+	})
+
+	return licenses
+}
+
+// extractImportedBy extracts the import count.
+func (p *Parser) extractImportedBy(doc *goquery.Document) int {
+	importEl := doc.Find("[data-test-id='UnitHeader-importedby'] a")
+	if importEl.Length() > 0 {
+		text := importEl.Text()
+		// Extract number from "Imported by: 144,729"
+		re := regexp.MustCompile(`[\d,]+`)
+		if match := re.FindString(text); match != "" {
+			match = strings.ReplaceAll(match, ",", "")
+			var count int
+			for _, c := range match {
+				if c >= '0' && c <= '9' {
+					count = count*10 + int(c-'0')
+				}
+			}
+			return count
+		}
+	}
+	return 0
+}
+
+// extractRepository extracts the repository URL.
+func (p *Parser) extractRepository(doc *goquery.Document) string {
+	repoEl := doc.Find(".UnitMeta-repo a")
+	if repoEl.Length() > 0 {
+		if href, exists := repoEl.Attr("href"); exists {
+			return href
+		}
+	}
+	return ""
+}
+
+// extractFunctions extracts all function declarations.
+func (p *Parser) extractFunctions(doc *goquery.Document) []*Function {
+	var functions []*Function
+
+	doc.Find(".Documentation-function").Each(func(_ int, s *goquery.Selection) {
+		fn := &Function{}
+
+		// Extract name from the function header
+		nameEl := s.Find(".Documentation-functionHeader").First()
+		if nameEl.Length() > 0 {
+			fn.Name = strings.TrimSpace(nameEl.Text())
+		}
+
+		// Extract signature from code block
+		sigEl := s.Find("pre").First()
+		if sigEl.Length() > 0 {
+			fn.Signature = strings.TrimSpace(sigEl.Text())
+		}
+
+		// Extract documentation
+		docEl := s.Find(".Documentation-functionBody p").First()
+		if docEl.Length() == 0 {
+			docEl = s.Find("p").First()
+		}
+		if docEl.Length() > 0 {
+			fn.Doc = strings.TrimSpace(docEl.Text())
+		}
+
+		// Extract examples
+		fn.Examples = p.extractExamples(s)
+
+		if fn.Name != "" {
+			functions = append(functions, fn)
+		}
+	})
+
+	return functions
+}
+
+// extractTypes extracts all type declarations.
+func (p *Parser) extractTypes(doc *goquery.Document) []*Type {
+	var types []*Type
+
+	doc.Find(".Documentation-type").Each(func(_ int, s *goquery.Selection) {
+		t := &Type{}
+
+		// Extract name from the type header
+		nameEl := s.Find(".Documentation-typeHeader").First()
+		if nameEl.Length() > 0 {
+			t.Name = strings.TrimSpace(nameEl.Text())
+		}
+
+		// Determine kind from signature
+		sigEl := s.Find("pre").First()
+		if sigEl.Length() > 0 {
+			sig := sigEl.Text()
+			t.Underlying = strings.TrimSpace(sig)
+
+			if strings.Contains(sig, "struct{") {
+				t.Kind = TypeKindStruct
+				t.Fields = p.extractStructFields(sigEl)
+			} else if strings.Contains(sig, "interface{") {
+				t.Kind = TypeKindInterface
+			} else {
+				t.Kind = TypeKindAlias
+			}
+		}
+
+		// Extract documentation
+		docEl := s.Find("p").First()
+		if docEl.Length() > 0 {
+			t.Doc = strings.TrimSpace(docEl.Text())
+		}
+
+		// Extract methods
+		t.Methods = p.extractMethods(s)
+
+		// Extract examples
+		t.Examples = p.extractExamples(s)
+
+		if t.Name != "" {
+			types = append(types, t)
+		}
+	})
+
+	return types
+}
+
+// extractStructFields extracts struct fields from a type definition.
+func (p *Parser) extractStructFields(sigEl *goquery.Selection) []*Field {
+	var fields []*Field
+
+	sigEl.Find("tr, .Documentation-structField").Each(func(_ int, s *goquery.Selection) {
+		text := s.Text()
+		text = strings.TrimSpace(text)
+
+		if text == "" || strings.HasPrefix(text, "//") {
+			return
+		}
+
+		// Parse field: Name Type `tag`
+		parts := strings.Fields(text)
+		if len(parts) >= 1 {
+			field := &Field{
+				Name:     parts[0],
+				Exported: isExported(parts[0]),
+			}
+
+			if len(parts) >= 2 {
+				field.Type = strings.Join(parts[1:], " ")
+				// Remove tag
+				if idx := strings.Index(field.Type, "`"); idx > 0 {
+					field.Tag = field.Type[idx:]
+					field.Type = field.Type[:idx]
+				}
+			}
+
+			fields = append(fields, field)
+		}
+	})
+
+	return fields
+}
+
+// extractMethods extracts methods from a type section.
+func (p *Parser) extractMethods(typeSection *goquery.Selection) []*Method {
+	var methods []*Method
+
+	typeSection.Find(".Documentation-method, .Documentation-function").Each(func(_ int, s *goquery.Selection) {
+		m := &Method{}
+
+		// Extract method name
+		nameEl := s.Find(".Documentation-functionHeader, .Documentation-methodHeader").First()
+		if nameEl.Length() > 0 {
+			name := strings.TrimSpace(nameEl.Text())
+			// Extract receiver if present: (t *Type) Method(...)
+			if strings.HasPrefix(name, "(") {
+				if end := strings.Index(name, ")"); end > 0 {
+					m.Receiver = name[1:end]
+					name = strings.TrimSpace(name[end+1:])
+				}
+			}
+			m.Name = name
+		}
+
+		// Extract signature
+		sigEl := s.Find("pre").First()
+		if sigEl.Length() > 0 {
+			m.Signature = strings.TrimSpace(sigEl.Text())
+		}
+
+		// Extract documentation
+		docEl := s.Find("p").First()
+		if docEl.Length() > 0 {
+			m.Doc = strings.TrimSpace(docEl.Text())
+		}
+
+		if m.Name != "" {
+			methods = append(methods, m)
+		}
+	})
+
+	return methods
+}
+
+// extractConstants extracts constant declarations.
+func (p *Parser) extractConstants(doc *goquery.Document) []*Value {
+	var constants []*Value
+
+	doc.Find(".Documentation-constants").Each(func(_ int, s *goquery.Selection) {
+		// Extract constant group
+		codeEl := s.Find("pre").First()
+		if codeEl.Length() > 0 {
+			v := &Value{
+				IsConst: true,
+			}
+
+			// Parse const declarations
+			text := codeEl.Text()
+			lines := strings.Split(text, "\n")
+
+			for _, line := range lines {
+				line = strings.TrimSpace(line)
+				if line == "" || strings.HasPrefix(line, "//") {
+					continue
+				}
+
+				// Simple const: Name = value
+				if strings.Contains(line, "=") {
+					parts := strings.SplitN(line, "=", 2)
+					if len(parts) == 2 {
+						name := strings.TrimSpace(parts[0])
+						if v.Names == nil {
+							v.Names = []string{}
+						}
+						v.Names = append(v.Names, name)
+						if v.Name == "" {
+							v.Name = name
+						}
+						v.Value = strings.TrimSpace(parts[1])
+					}
+				}
+			}
+
+			// Extract documentation
+			docEl := s.Find("p").First()
+			if docEl.Length() > 0 {
+				v.Doc = strings.TrimSpace(docEl.Text())
+			}
+
+			if len(v.Names) > 0 {
+				constants = append(constants, v)
+			}
+		}
+	})
+
+	return constants
+}
+
+// extractVariables extracts variable declarations.
+func (p *Parser) extractVariables(doc *goquery.Document) []*Value {
+	var variables []*Value
+
+	doc.Find(".Documentation-variables").Each(func(_ int, s *goquery.Selection) {
+		codeEl := s.Find("pre").First()
+		if codeEl.Length() > 0 {
+			v := &Value{
+				IsConst: false,
+			}
+
+			text := codeEl.Text()
+			// Parse var declarations
+			if strings.HasPrefix(text, "var ") {
+				text = strings.TrimPrefix(text, "var ")
+			}
+
+			lines := strings.Split(text, "\n")
+			for _, line := range lines {
+				line = strings.TrimSpace(line)
+				if line == "" {
+					continue
+				}
+
+				// Parse: Name Type = value
+				if strings.Contains(line, "=") {
+					parts := strings.SplitN(line, "=", 2)
+					if len(parts) == 2 {
+						nameType := strings.TrimSpace(parts[0])
+						v.Name = strings.Fields(nameType)[0]
+						v.Value = strings.TrimSpace(parts[1])
+						break
+					}
+				} else {
+					// Just name and type
+					fields := strings.Fields(line)
+					if len(fields) >= 1 {
+						v.Name = fields[0]
+						if len(fields) >= 2 {
+							v.Type = strings.Join(fields[1:], " ")
+						}
+					}
+				}
+			}
+
+			// Extract documentation
+			docEl := s.Find("p").First()
+			if docEl.Length() > 0 {
+				v.Doc = strings.TrimSpace(docEl.Text())
+			}
+
+			if v.Name != "" {
+				variables = append(variables, v)
+			}
+		}
+	})
+
+	return variables
+}
+
+// extractExamples extracts examples from a section.
+func (p *Parser) extractExamples(section *goquery.Selection) []*Example {
+	var examples []*Example
+
+	section.Find(".Documentation-example").Each(func(_ int, s *goquery.Selection) {
+		ex := &Example{}
+
+		// Extract example name
+		nameEl := s.Find(".Documentation-exampleHeader").First()
+		if nameEl.Length() > 0 {
+			ex.Name = strings.TrimSpace(nameEl.Text())
+		}
+
+		// Extract code
+		codeEl := s.Find("pre, code").First()
+		if codeEl.Length() > 0 {
+			ex.Code = strings.TrimSpace(codeEl.Text())
+		}
+
+		// Extract output
+		outputEl := s.Find(".Documentation-exampleOutput").First()
+		if outputEl.Length() > 0 {
+			ex.Output = strings.TrimSpace(outputEl.Text())
+		}
+
+		// Extract documentation
+		docEl := s.Find("p").First()
+		if docEl.Length() > 0 {
+			ex.Doc = strings.TrimSpace(docEl.Text())
+		}
+
+		if ex.Code != "" {
+			examples = append(examples, ex)
+		}
+	})
+
+	return examples
+}
+
+// extractPackageExamples extracts package-level examples.
+func (p *Parser) extractPackageExamples(doc *goquery.Document) []*Example {
+	var examples []*Example
+
+	doc.Find(".Documentation-example").Each(func(_ int, s *goquery.Selection) {
+		ex := &Example{}
+
+		// Extract example name
+		nameEl := s.Find(".Documentation-exampleHeader").First()
+		if nameEl.Length() > 0 {
+			ex.Name = strings.TrimSpace(nameEl.Text())
+		}
+
+		// Extract code
+		codeEl := s.Find("pre, code").First()
+		if codeEl.Length() > 0 {
+			ex.Code = strings.TrimSpace(codeEl.Text())
+		}
+
+		// Extract output
+		outputEl := s.Find(".Documentation-exampleOutput").First()
+		if outputEl.Length() > 0 {
+			ex.Output = strings.TrimSpace(outputEl.Text())
+		}
+
+		if ex.Code != "" {
+			examples = append(examples, ex)
+		}
+	})
+
+	return examples
+}
+
+// parseCount parses a count string to int.
+func parseCount(s string) int {
+	var count int
+	for _, c := range s {
+		if c >= '0' && c <= '9' {
+			count = count*10 + int(c-'0')
+		}
+	}
+	return count
+}
+
+// isExported checks if a name is exported (starts with uppercase).
+func isExported(name string) bool {
+	if len(name) == 0 {
+		return false
+	}
+	return name[0] >= 'A' && name[0] <= 'Z'
+}
+
+// cleanWhitespace normalizes whitespace in text.
+func cleanWhitespace(text string) string {
+	// Replace multiple whitespace with single space
+	re := regexp.MustCompile(`\s+`)
+	text = re.ReplaceAllString(text, " ")
+	return strings.TrimSpace(text)
+}

pkg/godocs/parser_test.go

@@ -0,0 +1,268 @@
+package godocs
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/PuerkitoBio/goquery"
+)
+
+const testPackageHTML = `
+<!DOCTYPE html>
+<html>
+<head><title>runtime - pkg.go.dev</title></head>
+<body>
+<nav class="go-Breadcrumb">
+  <ol>
+    <li><a href="/">Discover Packages</a></li>
+    <li><a href="/k8s.io/apimachinery">k8s.io/apimachinery</a></li>
+    <li><a href="/k8s.io/apimachinery/pkg">pkg</a></li>
+    <li><a href="/k8s.io/apimachinery/pkg/runtime">runtime</a></li>
+  </ol>
+</nav>
+
+<h1 class="UnitHeader-titleHeading">runtime</h1>
+
+<div class="go-Main-headerDetails">
+  <span data-test-id="UnitHeader-version"><a href="?tab=versions">v0.35.1</a></span>
+  <span data-test-id="UnitHeader-importedby"><a href="?tab=importedby">Imported by: 144,729</a></span>
+  <span data-test-id="UnitHeader-licenses"><a href="?tab=licenses">Apache-2.0</a></span>
+</div>
+
+<div class="Documentation">
+  <p>Package runtime defines conversions between generic types and structs to map query strings to struct objects.</p>
+  <p>This is additional documentation text for the package.</p>
+</div>
+
+<div class="Documentation-function">
+  <div class="Documentation-functionHeader">func DecodeInto</div>
+  <pre>func DecodeInto(d Decoder, data []byte, into Object) error</pre>
+  <p>DecodeInto is a helper function that decodes the given data into the provided object.</p>
+</div>
+
+<div class="Documentation-type">
+  <div class="Documentation-typeHeader">type Codec</div>
+  <pre>type Codec struct {
+    Encoder Encoder
+    Decoder Decoder
+}</pre>
+  <p>Codec is a struct that holds an encoder and decoder.</p>
+  <div class="Documentation-method">
+    <div class="Documentation-methodHeader">func (*Codec) Encode</div>
+    <pre>func (c *Codec) Encode(obj Object) ([]byte, error)</pre>
+    <p>Encode encodes the given object.</p>
+  </div>
+</div>
+
+<div class="Documentation-constants">
+  <pre>const (
+    ContentTypeJSON = "application/json"
+    ContentTypeYAML = "application/yaml"
+)</pre>
+  <p>Content types for different formats.</p>
+</div>
+
+<div class="Documentation-variables">
+  <pre>var DefaultScheme = NewScheme()</pre>
+  <p>DefaultScheme is the default scheme used for encoding/decoding.</p>
+</div>
+
+</body>
+</html>
+`
+
+const testSearchHTML = `
+<!DOCTYPE html>
+<html>
+<body>
+<div class="SearchSnippet">
+  <h2><a href="/k8s.io/apimachinery/pkg/runtime">runtime <span class="SearchSnippet-header-path">(k8s.io/apimachinery/pkg/runtime)</span></a></h2>
+  <p class="SearchSnippet-synopsis">Package runtime defines conversions between generic types and structs.</p>
+  <div class="SearchSnippet-infoLabel">
+    <a href="?tab=importedby">Imported by: <strong>144,729</strong></a>
+    <span>v0.35.1 published on <strong>Dec 4, 2025</strong></span>
+    <a href="?tab=licenses">Apache-2.0</a>
+  </div>
+</div>
+
+<div class="SearchSnippet">
+  <h2><a href="/github.com/google/go-querystring/query">query <span class="SearchSnippet-header-path">(github.com/google/go-querystring/query)</span></a></h2>
+  <p class="SearchSnippet-synopsis">Package query implements encoding of structs into URL query parameters.</p>
+  <div class="SearchSnippet-infoLabel">
+    <a href="?tab=importedby">Imported by: <strong>5,111</strong></a>
+    <span>v1.2.0 published on <strong>Nov 10, 2025</strong></span>
+    <a href="?tab=licenses">BSD-3-Clause</a>
+  </div>
+</div>
+</body>
+</html>
+`
+
+func TestParsePackagePage(t *testing.T) {
+	parser := NewParser()
+	pkg, err := parser.ParsePackagePage(testPackageHTML, "https://pkg.go.dev/k8s.io/apimachinery/pkg/runtime")
+	if err != nil {
+		t.Fatalf("ParsePackagePage failed: %v", err)
+	}
+
+	if pkg.Name != "runtime" {
+		t.Errorf("Expected name 'runtime', got '%s'", pkg.Name)
+	}
+
+	if pkg.ImportPath != "k8s.io/apimachinery/pkg/runtime" {
+		t.Errorf("Expected import path 'k8s.io/apimachinery/pkg/runtime', got '%s'", pkg.ImportPath)
+	}
+
+	if pkg.Version != "v0.35.1" {
+		t.Errorf("Expected version 'v0.35.1', got '%s'", pkg.Version)
+	}
+
+	if pkg.ImportedBy != 144729 {
+		t.Errorf("Expected imported by 144729, got %d", pkg.ImportedBy)
+	}
+
+	if pkg.Synopsis == "" {
+		t.Error("Expected non-empty synopsis")
+	}
+
+	if len(pkg.Functions) == 0 {
+		t.Error("Expected at least one function")
+	}
+
+	if len(pkg.Types) == 0 {
+		t.Error("Expected at least one type")
+	}
+
+	if len(pkg.Constants) == 0 {
+		t.Error("Expected at least one constant")
+	}
+
+	if len(pkg.Variables) == 0 {
+		t.Error("Expected at least one variable")
+	}
+}
+
+func TestParseSearchResults(t *testing.T) {
+	parser := NewParser()
+	results, err := parser.ParseSearchResults(testSearchHTML)
+	if err != nil {
+		t.Fatalf("ParseSearchResults failed: %v", err)
+	}
+
+	if len(results) < 2 {
+		t.Fatalf("Expected at least 2 results, got %d", len(results))
+	}
+
+	first := results[0]
+	if first.Synopsis == "" {
+		t.Error("Expected non-empty synopsis")
+	}
+
+	if first.Path == "" {
+		t.Error("Expected non-empty path")
+	}
+
+	if first.URL == "" {
+		t.Error("Expected non-empty URL")
+	}
+}
+
+func TestIsExported(t *testing.T) {
+	tests := []struct {
+		name     string
+		expected bool
+	}{
+		{"Exported", true},
+		{"unexported", false},
+		{"", false},
+		{"CamelCase", true},
+		{"camelCase", false},
+		{"X", true},
+		{"x", false},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := isExported(tt.name); got != tt.expected {
+				t.Errorf("isExported(%q) = %v, want %v", tt.name, got, tt.expected)
+			}
+		})
+	}
+}
+
+func TestCleanWhitespace(t *testing.T) {
+	tests := []struct {
+		input    string
+		expected string
+	}{
+		{"  hello   world  ", "hello world"},
+		{"single", "single"},
+		{"multiple   spaces   here", "multiple spaces here"},
+		{"\n\ttabs\t\n", "tabs"},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.input, func(t *testing.T) {
+			if got := cleanWhitespace(tt.input); got != tt.expected {
+				t.Errorf("cleanWhitespace(%q) = %q, want %q", tt.input, got, tt.expected)
+			}
+		})
+	}
+}
+
+func TestParseCount(t *testing.T) {
+	tests := []struct {
+		input    string
+		expected int
+	}{
+		{"144729", 144729},
+		{"5,111", 5111},
+		{"0", 0},
+		{"1,234,567", 1234567},
+		{"abc", 0},
+		{"", 0},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.input, func(t *testing.T) {
+			if got := parseCount(tt.input); got != tt.expected {
+				t.Errorf("parseCount(%q) = %d, want %d", tt.input, got, tt.expected)
+			}
+		})
+	}
+}
+
+func TestExtractImportPath(t *testing.T) {
+	parser := NewParser()
+
+	tests := []struct {
+		html     string
+		url      string
+		expected string
+	}{
+		{
+			html:     `<nav class="go-Breadcrumb"><li><a href="/">Discover</a></li><li><a href="/k8s.io/apimachinery">k8s.io/apimachinery</a></li><li><a href="/k8s.io/apimachinery/pkg">pkg</a></li><li><a href="/k8s.io/apimachinery/pkg/runtime">runtime</a></li></nav>`,
+			url:      "https://pkg.go.dev/k8s.io/apimachinery/pkg/runtime",
+			expected: "k8s.io/apimachinery/pkg/runtime",
+		},
+		{
+			html:     `<nav class="go-Breadcrumb"><li><a href="/github.com/user/repo">github.com/user/repo</a></li></nav>`,
+			url:      "https://pkg.go.dev/github.com/user/[email protected]",
+			expected: "github.com/user/repo",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.expected, func(t *testing.T) {
+			doc, err := goquery.NewDocumentFromReader(strings.NewReader(tt.html))
+			if err != nil {
+				t.Fatalf("Failed to parse HTML: %v", err)
+			}
+
+			got := parser.extractImportPath(doc, tt.url)
+			if got != tt.expected {
+				t.Errorf("extractImportPath() = %q, want %q", got, tt.expected)
+			}
+		})
+	}
+}

pkg/godocs/types.go

@@ -0,0 +1,156 @@
+// Package godocs provides parsing and extraction for Go package documentation
+// from pkg.go.dev and similar documentation sites.
+package godocs
+
+import "time"
+
+// Package represents a Go package's documentation.
+type Package struct {
+	// Import path (e.g., "github.com/user/repo/pkg")
+	ImportPath string `json:"import_path"`
+
+	// Package name (last element of import path)
+	Name string `json:"name"`
+
+	// Synopsis is a short one-line description
+	Synopsis string `json:"synopsis"`
+
+	// Full documentation text
+	Doc string `json:"doc"`
+
+	// Version information
+	Version string `json:"version"`
+
+	// Module information
+	Module *Module `json:"module,omitempty"`
+
+	// License information
+	Licenses []License `json:"licenses,omitempty"`
+
+	// Functions exported by the package
+	Functions []*Function `json:"functions,omitempty"`
+
+	// Types defined in the package
+	Types []*Type `json:"types,omitempty"`
+
+	// Constants defined in the package
+	Constants []*Value `json:"constants,omitempty"`
+
+	// Variables defined in the package
+	Variables []*Value `json:"variables,omitempty"`
+
+	// Examples for the package
+	Examples []*Example `json:"examples,omitempty"`
+
+	// Import count
+	ImportedBy int `json:"imported_by"`
+
+	// Repository URL
+	Repository string `json:"repository,omitempty"`
+
+	// Documentation URL
+	DocURL string `json:"doc_url"`
+
+	// When the documentation was fetched
+	FetchedAt time.Time `json:"fetched_at"`
+}
+
+// Module represents Go module information.
+type Module struct {
+	Path    string `json:"path"`
+	Version string `json:"version"`
+}
+
+// License represents license information.
+type License struct {
+	Name string `json:"name"`
+	Path string `json:"path,omitempty"`
+}
+
+// Function represents a function declaration.
+type Function struct {
+	Name      string     `json:"name"`
+	Doc       string     `json:"doc,omitempty"`
+	Signature string     `json:"signature"`
+	Examples  []*Example `json:"examples,omitempty"`
+}
+
+// Type represents a type declaration.
+type Type struct {
+	Name       string     `json:"name"`
+	Doc        string     `json:"doc,omitempty"`
+	Kind       TypeKind   `json:"kind"`
+	Underlying string     `json:"underlying,omitempty"` // For type aliases
+	Fields     []*Field   `json:"fields,omitempty"`     // For structs
+	Methods    []*Method  `json:"methods,omitempty"`
+	Examples   []*Example `json:"examples,omitempty"`
+}
+
+// TypeKind represents the kind of type.
+type TypeKind string
+
+const (
+	TypeKindBasic     TypeKind = "basic"
+	TypeKindStruct    TypeKind = "struct"
+	TypeKindInterface TypeKind = "interface"
+	TypeKindAlias     TypeKind = "alias"
+	TypeKindFunc      TypeKind = "func"
+)
+
+// Field represents a struct field.
+type Field struct {
+	Name     string `json:"name"`
+	Type     string `json:"type"`
+	Doc      string `json:"doc,omitempty"`
+	Tag      string `json:"tag,omitempty"`
+	Embedded bool   `json:"embedded,omitempty"`
+	Exported bool   `json:"exported"`
+}
+
+// Method represents a method on a type.
+type Method struct {
+	Name      string `json:"name"`
+	Doc       string `json:"doc,omitempty"`
+	Signature string `json:"signature"`
+	Receiver  string `json:"receiver,omitempty"`
+}
+
+// Value represents a constant or variable declaration.
+type Value struct {
+	Name    string   `json:"name"`
+	Doc     string   `json:"doc,omitempty"`
+	Type    string   `json:"type,omitempty"`
+	Value   string   `json:"value,omitempty"`
+	Names   []string `json:"names,omitempty"` // For const groups
+	IsConst bool     `json:"is_const"`
+}
+
+// Example represents a code example.
+type Example struct {
+	Name    string `json:"name"`
+	Doc     string `json:"doc,omitempty"`
+	Code    string `json:"code"`
+	Output  string `json:"output,omitempty"`
+	PlayURL string `json:"play_url,omitempty"`
+}
+
+// SearchResult represents a search result from pkg.go.dev.
+type SearchResult struct {
+	Name       string `json:"name"`
+	Path       string `json:"path"`
+	Synopsis   string `json:"synopsis"`
+	ImportedBy int    `json:"imported_by"`
+	Version    string `json:"version"`
+	Published  string `json:"published"`
+	License    string `json:"license"`
+	URL        string `json:"url"`
+}
+
+// Symbol represents a symbol (function, type, etc.) within a package.
+type Symbol struct {
+	Name      string `json:"name"`
+	Kind      string `json:"kind"` // function, type, constant, variable
+	Signature string `json:"signature,omitempty"`
+	Doc       string `json:"doc,omitempty"`
+	Recv      string `json:"recv,omitempty"` // For methods
+}

pkg/javadocs/parser.go

@@ -0,0 +1,369 @@
+package javadocs
+
+import (
+	"net/url"
+	"strings"
+	"time"
+
+	"github.com/PuerkitoBio/goquery"
+)
+
+type Parser struct {
+	baseURL string
+}
+
+func NewParser() *Parser {
+	return &Parser{
+		baseURL: "https://docs.oracle.com",
+	}
+}
+
+func (p *Parser) ParsePackagePage(html string, docURL string) (*Package, error) {
+	doc, err := goquery.NewDocumentFromReader(strings.NewReader(html))
+	if err != nil {
+		return nil, err
+	}
+
+	pkg := &Package{
+		DocURL:    docURL,
+		FetchedAt: time.Now(),
+	}
+
+	pkg.Name = p.extractPackageName(doc)
+	pkg.Doc = p.extractPackageDoc(doc)
+	pkg.Classes = p.extractClasses(doc, pkg.Name, docURL)
+	pkg.Interfaces = p.extractInterfaces(doc, pkg.Name, docURL)
+	pkg.Enums = p.extractEnums(doc, pkg.Name, docURL)
+	pkg.Exceptions = p.extractExceptions(doc, pkg.Name, docURL)
+
+	return pkg, nil
+}
+
+func (p *Parser) ParseSearchResults(html string) ([]*SearchResult, error) {
+	doc, err := goquery.NewDocumentFromReader(strings.NewReader(html))
+	if err != nil {
+		return nil, err
+	}
+
+	var results []*SearchResult
+
+	doc.Find(".result").Each(func(i int, s *goquery.Selection) {
+		result := &SearchResult{}
+
+		link := s.Find("a").First()
+		result.Name = strings.TrimSpace(link.Text())
+
+		if href, exists := link.Attr("href"); exists {
+			result.DocURL = resolveURL(p.baseURL, href)
+		}
+
+		result.Kind = s.Find(".result-kind").Text()
+		result.QualName = s.Find(".qualified-name").Text()
+		result.Package = s.Find(".package").Text()
+		result.Doc = strings.TrimSpace(s.Find(".description").Text())
+
+		results = append(results, result)
+	})
+
+	return results, nil
+}
+
+func (p *Parser) extractPackageName(doc *goquery.Document) string {
+	title := doc.Find("h1, .title").First().Text()
+	title = strings.TrimSpace(title)
+
+	if strings.Contains(title, "Package") {
+		parts := strings.Fields(title)
+		for i, part := range parts {
+			if part == "Package" && i+1 < len(parts) {
+				return parts[i+1]
+			}
+		}
+	}
+
+	if title != "" {
+		return title
+	}
+
+	return ""
+}
+
+func (p *Parser) extractPackageDoc(doc *goquery.Document) string {
+	docblock := doc.Find(".block, .description, #package-description").First()
+	return strings.TrimSpace(docblock.Text())
+}
+
+func (p *Parser) extractClasses(doc *goquery.Document, pkgName string, docURL string) []*Class {
+	var classes []*Class
+
+	doc.Find("table.type-summary tr, .class-summary .member, section.class tbody tr").Each(func(_ int, s *goquery.Selection) {
+		class := &Class{
+			Package: pkgName,
+			Kind:    ClassKindClass,
+		}
+
+		link := s.Find("a").First()
+		class.Name = strings.TrimSpace(link.Text())
+
+		if class.Name == "" {
+			class.Name = strings.TrimSpace(s.Find(".member-name, td:first-child").Text())
+		}
+
+		if href, exists := link.Attr("href"); exists {
+			class.DocURL = resolveURL(docURL, href)
+			class.QualifiedName = pkgName + "." + class.Name
+		}
+
+		class.Doc = strings.TrimSpace(s.Find(".member-summary, td:last-child").Text())
+
+		if class.Name != "" && !strings.Contains(class.Name, "interface") {
+			classes = append(classes, class)
+		}
+	})
+
+	return classes
+}
+
+func (p *Parser) extractInterfaces(doc *goquery.Document, pkgName string, docURL string) []*Class {
+	var interfaces []*Class
+
+	doc.Find("table.interface-summary tr, .interface-summary .member").Each(func(_ int, s *goquery.Selection) {
+		iface := &Class{
+			Package: pkgName,
+			Kind:    ClassKindInterface,
+		}
+
+		link := s.Find("a").First()
+		iface.Name = strings.TrimSpace(link.Text())
+
+		if iface.Name == "" {
+			iface.Name = strings.TrimSpace(s.Find(".member-name").Text())
+		}
+
+		if href, exists := link.Attr("href"); exists {
+			iface.DocURL = resolveURL(docURL, href)
+			iface.QualifiedName = pkgName + "." + iface.Name
+		}
+
+		iface.Doc = strings.TrimSpace(s.Find(".member-summary, td:last-child").Text())
+
+		if iface.Name != "" {
+			interfaces = append(interfaces, iface)
+		}
+	})
+
+	return interfaces
+}
+
+func (p *Parser) extractEnums(doc *goquery.Document, pkgName string, docURL string) []*Enum {
+	var enums []*Enum
+
+	doc.Find("table.enum-summary tr, .enum-summary .member").Each(func(_ int, s *goquery.Selection) {
+		enum := &Enum{
+			Package: pkgName,
+		}
+
+		link := s.Find("a").First()
+		enum.Name = strings.TrimSpace(link.Text())
+
+		if enum.Name == "" {
+			enum.Name = strings.TrimSpace(s.Find(".member-name").Text())
+		}
+
+		if href, exists := link.Attr("href"); exists {
+			enum.DocURL = resolveURL(docURL, href)
+			enum.QualifiedName = pkgName + "." + enum.Name
+		}
+
+		enum.Doc = strings.TrimSpace(s.Find(".member-summary, td:last-child").Text())
+
+		if enum.Name != "" {
+			enums = append(enums, enum)
+		}
+	})
+
+	return enums
+}
+
+func (p *Parser) extractExceptions(doc *goquery.Document, pkgName string, docURL string) []*Class {
+	var exceptions []*Class
+
+	doc.Find("table.exception-summary tr, .exception-summary .member").Each(func(_ int, s *goquery.Selection) {
+		exc := &Class{
+			Package: pkgName,
+			Kind:    ClassKindClass,
+		}
+
+		link := s.Find("a").First()
+		exc.Name = strings.TrimSpace(link.Text())
+
+		if exc.Name == "" {
+			exc.Name = strings.TrimSpace(s.Find(".member-name").Text())
+		}
+
+		if href, exists := link.Attr("href"); exists {
+			exc.DocURL = resolveURL(docURL, href)
+			exc.QualifiedName = pkgName + "." + exc.Name
+		}
+
+		exc.Doc = strings.TrimSpace(s.Find(".member-summary, td:last-child").Text())
+
+		if exc.Name != "" {
+			exceptions = append(exceptions, exc)
+		}
+	})
+
+	return exceptions
+}
+
+func (p *Parser) ParseClassPage(html string, docURL string) (*Class, error) {
+	doc, err := goquery.NewDocumentFromReader(strings.NewReader(html))
+	if err != nil {
+		return nil, err
+	}
+
+	class := &Class{
+		DocURL: docURL,
+	}
+
+	header := doc.Find(".header, h1, .class-name").First()
+	class.Name = strings.TrimSpace(header.Text())
+
+	class.QualifiedName = class.Name
+	if idx := strings.LastIndex(class.Name, "."); idx > 0 {
+		class.Package = class.Name[:idx]
+		class.Name = class.Name[idx+1:]
+	}
+
+	class.Doc = strings.TrimSpace(doc.Find(".block, .description, .class-description").First().Text())
+
+	class.Methods = p.extractMethods(doc, class.Name, docURL)
+	class.Fields = p.extractFields(doc, class.Name, docURL)
+	class.Constructors = p.extractConstructors(doc, class.Name, docURL)
+
+	return class, nil
+}
+
+func (p *Parser) extractMethods(doc *goquery.Document, className string, docURL string) []*Method {
+	var methods []*Method
+
+	doc.Find("table.method-summary tr, .method-summary .member, section.method-detail > ul > li").Each(func(_ int, s *goquery.Selection) {
+		method := &Method{
+			IsConstructor: false,
+		}
+
+		link := s.Find("a").First()
+		method.Name = strings.TrimSpace(link.Text())
+
+		if method.Name == "" {
+			sig := s.Find(".member-signature, code").Text()
+			method.Name = extractMethodName(sig)
+		}
+
+		sigEl := s.Find(".member-signature, code, .sig")
+		method.Signature = strings.TrimSpace(sigEl.Text())
+
+		if id, exists := s.Attr("id"); exists {
+			method.DocURL = docURL + "#" + id
+			method.QualifiedName = className + "." + method.Name
+		} else if href, exists := link.Attr("href"); exists {
+			method.DocURL = resolveURL(docURL, href)
+			method.QualifiedName = className + "." + method.Name
+		}
+
+		method.Doc = strings.TrimSpace(s.Find(".block, .member-summary, dd").First().Text())
+
+		if method.Name != "" {
+			methods = append(methods, method)
+		}
+	})
+
+	return methods
+}
+
+func (p *Parser) extractFields(doc *goquery.Document, className string, docURL string) []*Field {
+	var fields []*Field
+
+	doc.Find("table.field-summary tr, .field-summary .member").Each(func(_ int, s *goquery.Selection) {
+		field := &Field{}
+
+		link := s.Find("a").First()
+		field.Name = strings.TrimSpace(link.Text())
+
+		if field.Name == "" {
+			field.Name = strings.TrimSpace(s.Find(".member-name, td:first-child").Text())
+		}
+
+		field.Type = strings.TrimSpace(s.Find(".member-type, td:nth-child(2)").Text())
+		field.Doc = strings.TrimSpace(s.Find(".member-summary, td:last-child").Text())
+
+		if id, exists := s.Attr("id"); exists {
+			field.DocURL = docURL + "#" + id
+		}
+
+		if field.Name != "" {
+			fields = append(fields, field)
+		}
+	})
+
+	return fields
+}
+
+func (p *Parser) extractConstructors(doc *goquery.Document, className string, docURL string) []*Method {
+	var constructors []*Method
+
+	doc.Find("table.constructor-summary tr, .constructor-summary .member").Each(func(_ int, s *goquery.Selection) {
+		ctor := &Method{
+			IsConstructor: true,
+			Name:          className,
+		}
+
+		link := s.Find("a").First()
+		if name := strings.TrimSpace(link.Text()); name != "" {
+			ctor.Name = name
+		}
+
+		sigEl := s.Find(".member-signature, code")
+		ctor.Signature = strings.TrimSpace(sigEl.Text())
+
+		ctor.Doc = strings.TrimSpace(s.Find(".block, .member-summary, td:last-child").Text())
+
+		if id, exists := s.Attr("id"); exists {
+			ctor.DocURL = docURL + "#" + id
+		}
+
+		constructors = append(constructors, ctor)
+	})
+
+	return constructors
+}
+
+func extractMethodName(sig string) string {
+	sig = strings.TrimSpace(sig)
+	if idx := strings.Index(sig, "("); idx > 0 {
+		prefix := sig[:idx]
+		parts := strings.Fields(prefix)
+		if len(parts) > 0 {
+			return parts[len(parts)-1]
+		}
+	}
+	return ""
+}
+
+func resolveURL(base string, href string) string {
+	if strings.HasPrefix(href, "http") {
+		return href
+	}
+
+	baseURL, err := url.Parse(base)
+	if err != nil {
+		return href
+	}
+
+	hrefURL, err := url.Parse(href)
+	if err != nil {
+		return href
+	}
+
+	return baseURL.ResolveReference(hrefURL).String()
+}

pkg/javadocs/parser_test.go

@@ -0,0 +1,115 @@
+package javadocs
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/PuerkitoBio/goquery"
+)
+
+const testPackagePageHTML = `
+<!DOCTYPE html>
+<html>
+<body>
+<h1>Package java.util</h1>
+<div class="block">Contains the collections framework, legacy collection classes, event model, date and time facilities.</div>
+
+<table class="type-summary">
+<tr>
+<td><a href="ArrayList.html">ArrayList</a></td>
+<td>Resizable-array implementation of the List interface.</td>
+</tr>
+<tr>
+<td><a href="HashMap.html">HashMap</a></td>
+<td>Hash table based implementation of the Map interface.</td>
+</tr>
+</table>
+
+<table class="interface-summary">
+<tr>
+<td><a href="List.html">List</a></td>
+<td>An ordered collection (also known as a sequence).</td>
+</tr>
+</table>
+
+<table class="exception-summary">
+<tr>
+<td><a href="ConcurrentModificationException.html">ConcurrentModificationException</a></td>
+<td>This exception may be thrown by methods that detect concurrent modification.</td>
+</tr>
+</table>
+</body>
+</html>
+`
+
+func TestParsePackagePage(t *testing.T) {
+	parser := NewParser()
+	pkg, err := parser.ParsePackagePage(testPackagePageHTML, "https://docs.oracle.com/en/java/javase/17/docs/api/java.base/java/util/package-summary.html")
+	if err != nil {
+		t.Fatalf("ParsePackagePage failed: %v", err)
+	}
+
+	if pkg.Name == "" {
+		t.Error("Expected non-empty package name")
+	}
+
+	if pkg.Doc == "" {
+		t.Error("Expected non-empty doc")
+	}
+
+	if len(pkg.Classes) == 0 {
+		t.Error("Expected at least one class")
+	}
+
+	if len(pkg.Interfaces) == 0 {
+		t.Error("Expected at least one interface")
+	}
+
+	if len(pkg.Exceptions) == 0 {
+		t.Error("Expected at least one exception")
+	}
+}
+
+func TestExtractClasses(t *testing.T) {
+	parser := NewParser()
+
+	doc, err := goquery.NewDocumentFromReader(strings.NewReader(testPackagePageHTML))
+	if err != nil {
+		t.Fatalf("Failed to parse HTML: %v", err)
+	}
+
+	classes := parser.extractClasses(doc, "java.util", "https://docs.oracle.com/test")
+
+	if len(classes) == 0 {
+		t.Fatal("Expected at least one class")
+	}
+
+	first := classes[0]
+	if first.Name == "" {
+		t.Error("Expected non-empty class name")
+	}
+
+	if first.Package != "java.util" {
+		t.Errorf("Expected package 'java.util', got %q", first.Package)
+	}
+}
+
+func TestResolveURL(t *testing.T) {
+	tests := []struct {
+		base     string
+		href     string
+		expected string
+	}{
+		{"https://docs.oracle.com", "/api/ArrayList.html", "https://docs.oracle.com/api/ArrayList.html"},
+		{"https://docs.oracle.com", "https://example.com/page", "https://example.com/page"},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.href, func(t *testing.T) {
+			got := resolveURL(tt.base, tt.href)
+			if got != tt.expected {
+				t.Errorf("resolveURL(%q, %q) = %q, want %q", tt.base, tt.href, got, tt.expected)
+			}
+		})
+	}
+}

pkg/javadocs/types.go

@@ -0,0 +1,116 @@
+// Package javadocs provides parsing and extraction for Java documentation
+// from docs.oracle.com and javadoc-generated sites.
+package javadocs
+
+import "time"
+
+// Package represents a Java package's documentation.
+type Package struct {
+	Name        string    `json:"name"`
+	Doc         string    `json:"doc,omitempty"`
+	Version     string    `json:"version,omitempty"`
+	DocURL      string    `json:"doc_url"`
+	Classes     []*Class  `json:"classes,omitempty"`
+	Interfaces  []*Class  `json:"interfaces,omitempty"`
+	Enums       []*Enum   `json:"enums,omitempty"`
+	Exceptions  []*Class  `json:"exceptions,omitempty"`
+	Annotations []*Class  `json:"annotations,omitempty"`
+	FetchedAt   time.Time `json:"fetched_at"`
+}
+
+// Class represents a Java class or interface.
+type Class struct {
+	QualifiedName string    `json:"qualified_name"`
+	Name          string    `json:"name"`
+	Package       string    `json:"package"`
+	Kind          ClassKind `json:"kind"`
+	Doc           string    `json:"doc,omitempty"`
+	Modifiers     []string  `json:"modifiers,omitempty"`
+	SuperClass    string    `json:"super_class,omitempty"`
+	Interfaces    []string  `json:"interfaces,omitempty"`
+	Fields        []*Field  `json:"fields,omitempty"`
+	Methods       []*Method `json:"methods,omitempty"`
+	Constructors  []*Method `json:"constructors,omitempty"`
+	NestedClasses []*Class  `json:"nested_classes,omitempty"`
+	DocURL        string    `json:"doc_url"`
+	Since         string    `json:"since,omitempty"`
+	Deprecated    string    `json:"deprecated,omitempty"`
+}
+
+// ClassKind represents the kind of class.
+type ClassKind string
+
+const (
+	ClassKindClass      ClassKind = "class"
+	ClassKindInterface  ClassKind = "interface"
+	ClassKindAnnotation ClassKind = "annotation"
+	ClassKindRecord     ClassKind = "record"
+	ClassKindSealed     ClassKind = "sealed"
+)
+
+// Enum represents a Java enum.
+type Enum struct {
+	QualifiedName string       `json:"qualified_name"`
+	Name          string       `json:"name"`
+	Package       string       `json:"package"`
+	Doc           string       `json:"doc,omitempty"`
+	Modifiers     []string     `json:"modifiers,omitempty"`
+	Constants     []*EnumConst `json:"constants,omitempty"`
+	Methods       []*Method    `json:"methods,omitempty"`
+	DocURL        string       `json:"doc_url"`
+	Since         string       `json:"since,omitempty"`
+}
+
+// EnumConst represents an enum constant.
+type EnumConst struct {
+	Name   string `json:"name"`
+	Doc    string `json:"doc,omitempty"`
+	DocURL string `json:"doc_url"`
+}
+
+// Field represents a class field.
+type Field struct {
+	Name       string   `json:"name"`
+	Type       string   `json:"type"`
+	Doc        string   `json:"doc,omitempty"`
+	Modifiers  []string `json:"modifiers,omitempty"`
+	Value      string   `json:"value,omitempty"`
+	DocURL     string   `json:"doc_url"`
+	Deprecated string   `json:"deprecated,omitempty"`
+}
+
+// Method represents a method or constructor.
+type Method struct {
+	Name          string       `json:"name"`
+	Doc           string       `json:"doc,omitempty"`
+	Modifiers     []string     `json:"modifiers,omitempty"`
+	ReturnType    string       `json:"return_type,omitempty"`
+	Parameters    []*Parameter `json:"parameters,omitempty"`
+	Exceptions    []string     `json:"exceptions,omitempty"`
+	Signature     string       `json:"signature"`
+	QualifiedName string       `json:"qualified_name"`
+	DocURL        string       `json:"doc_url"`
+	Since         string       `json:"since,omitempty"`
+	Deprecated    string       `json:"deprecated,omitempty"`
+	IsConstructor bool         `json:"is_constructor"`
+	IsStatic      bool         `json:"is_static"`
+	IsDefault     bool         `json:"is_default"`
+}
+
+// Parameter represents a method parameter.
+type Parameter struct {
+	Name string `json:"name"`
+	Type string `json:"type"`
+	Doc  string `json:"doc,omitempty"`
+}
+
+// SearchResult represents a search result.
+type SearchResult struct {
+	Name     string `json:"name"`
+	QualName string `json:"qualified_name"`
+	Kind     string `json:"kind"` // class, interface, enum, method, field
+	Package  string `json:"package"`
+	Doc      string `json:"doc,omitempty"`
+	DocURL   string `json:"doc_url"`
+	Score    int    `json:"score"`
+}

pkg/mcpdocs/parser.go

@@ -0,0 +1,212 @@
+package mcpdocs
+
+import (
+	"net/url"
+	"strings"
+	"time"
+
+	"github.com/PuerkitoBio/goquery"
+)
+
+type Parser struct {
+	baseURL string
+}
+
+func NewParser() *Parser {
+	return &Parser{
+		baseURL: "https://hub.docker.com",
+	}
+}
+
+func (p *Parser) ParseServerPage(html string, docURL string) (*Server, error) {
+	doc, err := goquery.NewDocumentFromReader(strings.NewReader(html))
+	if err != nil {
+		return nil, err
+	}
+
+	server := &Server{
+		DocURL:    docURL,
+		FetchedAt: time.Now(),
+	}
+
+	server.Name = p.extractServerName(doc)
+	server.Description = p.extractDescription(doc)
+	server.Image = p.extractImage(doc)
+	server.Category = p.extractCategory(doc)
+	server.Tools = p.extractTools(doc, docURL)
+	server.Resources = p.extractResources(doc, docURL)
+	server.Prompts = p.extractPrompts(doc, docURL)
+
+	return server, nil
+}
+
+func (p *Parser) ParseHubPage(html string) ([]*Server, error) {
+	doc, err := goquery.NewDocumentFromReader(strings.NewReader(html))
+	if err != nil {
+		return nil, err
+	}
+
+	var servers []*Server
+
+	doc.Find("a[href*='/mcp/server/'], .server-card, .mcp-server-item").Each(func(_ int, s *goquery.Selection) {
+		server := &Server{}
+
+		server.Name = strings.TrimSpace(s.Find("h1, h2, h3, .name, .title").First().Text())
+		server.Description = strings.TrimSpace(s.Find(".description, p").First().Text())
+
+		if href, exists := s.Attr("href"); exists {
+			server.DocURL = resolveURL(p.baseURL, href)
+		}
+
+		if server.Name != "" {
+			servers = append(servers, server)
+		}
+	})
+
+	return servers, nil
+}
+
+func (p *Parser) extractServerName(doc *goquery.Document) string {
+	title := doc.Find("h1").First().Text()
+	title = strings.TrimSpace(title)
+
+	if title == "" {
+		title = doc.Find("title").First().Text()
+		title = strings.TrimSpace(title)
+		if idx := strings.Index(title, " | "); idx > 0 {
+			title = title[:idx]
+		}
+	}
+
+	return title
+}
+
+func (p *Parser) extractDescription(doc *goquery.Document) string {
+	desc := doc.Find("meta[name='description']").AttrOr("content", "")
+	if desc != "" {
+		return desc
+	}
+
+	desc = doc.Find(".description, .overview, .introduction, p:first-of-type").First().Text()
+	return strings.TrimSpace(desc)
+}
+
+func (p *Parser) extractImage(doc *goquery.Document) string {
+	return doc.Find("meta[property='og:image']").AttrOr("content", "")
+}
+
+func (p *Parser) extractCategory(doc *goquery.Document) string {
+	return doc.Find(".category, .tag").First().Text()
+}
+
+func (p *Parser) extractTools(doc *goquery.Document, docURL string) []*Tool {
+	var tools []*Tool
+
+	doc.Find("h2:contains('Tools'), h3:contains('Tools')").Each(func(_ int, heading *goquery.Selection) {
+		container := heading.Next()
+		for container.Length() > 0 && !container.Is("h2, h3") {
+			container.Find("li, .tool, .item").Each(func(_ int, item *goquery.Selection) {
+				tool := &Tool{}
+
+				tool.Name = strings.TrimSpace(item.Find("code, .name, strong").First().Text())
+				tool.Description = strings.TrimSpace(item.Find(".description, p").First().Text())
+				tool.DocURL = docURL
+
+				if tool.Name != "" {
+					tools = append(tools, tool)
+				}
+			})
+			container = container.Next()
+		}
+	})
+
+	doc.Find("pre code, .code-block").Each(func(_ int, code *goquery.Selection) {
+		text := code.Text()
+		if strings.Contains(text, "tools") && strings.Contains(text, "name") {
+			lines := strings.Split(text, "\n")
+			for _, line := range lines {
+				line = strings.TrimSpace(line)
+				if strings.HasPrefix(line, "name:") || strings.Contains(line, `"name"`) {
+					tool := &Tool{
+						DocURL: docURL,
+					}
+					parts := strings.SplitN(line, ":", 2)
+					if len(parts) > 1 {
+						tool.Name = strings.Trim(strings.TrimSpace(parts[1]), `"`)
+					}
+					if tool.Name != "" {
+						tools = append(tools, tool)
+					}
+				}
+			}
+		}
+	})
+
+	return tools
+}
+
+func (p *Parser) extractResources(doc *goquery.Document, docURL string) []*Resource {
+	var resources []*Resource
+
+	doc.Find("h2:contains('Resources'), h3:contains('Resources')").Each(func(_ int, heading *goquery.Selection) {
+		container := heading.Next()
+		for container.Length() > 0 && !container.Is("h2, h3") {
+			container.Find("li, .resource, .item").Each(func(_ int, item *goquery.Selection) {
+				res := &Resource{}
+
+				res.Name = strings.TrimSpace(item.Find("code, .name, strong").First().Text())
+				res.Description = strings.TrimSpace(item.Find(".description, p").First().Text())
+				res.DocURL = docURL
+
+				if res.Name != "" {
+					resources = append(resources, res)
+				}
+			})
+			container = container.Next()
+		}
+	})
+
+	return resources
+}
+
+func (p *Parser) extractPrompts(doc *goquery.Document, docURL string) []*Prompt {
+	var prompts []*Prompt
+
+	doc.Find("h2:contains('Prompts'), h3:contains('Prompts')").Each(func(_ int, heading *goquery.Selection) {
+		container := heading.Next()
+		for container.Length() > 0 && !container.Is("h2, h3") {
+			container.Find("li, .prompt, .item").Each(func(_ int, item *goquery.Selection) {
+				prompt := &Prompt{}
+
+				prompt.Name = strings.TrimSpace(item.Find("code, .name, strong").First().Text())
+				prompt.Description = strings.TrimSpace(item.Find(".description, p").First().Text())
+				prompt.DocURL = docURL
+
+				if prompt.Name != "" {
+					prompts = append(prompts, prompt)
+				}
+			})
+			container = container.Next()
+		}
+	})
+
+	return prompts
+}
+
+func resolveURL(base string, href string) string {
+	if strings.HasPrefix(href, "http") {
+		return href
+	}
+
+	baseURL, err := url.Parse(base)
+	if err != nil {
+		return href
+	}
+
+	hrefURL, err := url.Parse(href)
+	if err != nil {
+		return href
+	}
+
+	return baseURL.ResolveReference(hrefURL).String()
+}

pkg/mcpdocs/types.go

@@ -0,0 +1,49 @@
+package mcpdocs
+
+import "time"
+
+type Server struct {
+	Name        string      `json:"name"`
+	Description string      `json:"description,omitempty"`
+	Image       string      `json:"image,omitempty"`
+	DocURL      string      `json:"doc_url"`
+	Category    string      `json:"category,omitempty"`
+	Tools       []*Tool     `json:"tools,omitempty"`
+	Resources   []*Resource `json:"resources,omitempty"`
+	Prompts     []*Prompt   `json:"prompts,omitempty"`
+	FetchedAt   time.Time   `json:"fetched_at"`
+}
+
+type Tool struct {
+	Name        string       `json:"name"`
+	Description string       `json:"description,omitempty"`
+	InputSchema *InputSchema `json:"input_schema,omitempty"`
+	DocURL      string       `json:"doc_url"`
+}
+
+type Resource struct {
+	URI         string `json:"uri"`
+	Name        string `json:"name"`
+	Description string `json:"description,omitempty"`
+	MimeType    string `json:"mime_type,omitempty"`
+	DocURL      string `json:"doc_url"`
+}
+
+type Prompt struct {
+	Name        string      `json:"name"`
+	Description string      `json:"description,omitempty"`
+	Arguments   []*Argument `json:"arguments,omitempty"`
+	DocURL      string      `json:"doc_url"`
+}
+
+type Argument struct {
+	Name        string `json:"name"`
+	Description string `json:"description,omitempty"`
+	Required    bool   `json:"required"`
+}
+
+type InputSchema struct {
+	Type       string                 `json:"type"`
+	Properties map[string]interface{} `json:"properties,omitempty"`
+	Required   []string               `json:"required,omitempty"`
+}

pkg/nuxtdocs/parser.go

@@ -0,0 +1,266 @@
+package nuxtdocs
+
+import (
+	"net/url"
+	"strings"
+	"time"
+
+	"github.com/PuerkitoBio/goquery"
+)
+
+type Parser struct {
+	baseURL string
+}
+
+func NewParser() *Parser {
+	return &Parser{
+		baseURL: "https://nuxt.com",
+	}
+}
+
+func (p *Parser) ParseReferencePage(html string, docURL string) (*Reference, error) {
+	doc, err := goquery.NewDocumentFromReader(strings.NewReader(html))
+	if err != nil {
+		return nil, err
+	}
+
+	ref := &Reference{
+		DocURL:    docURL,
+		FetchedAt: time.Now(),
+	}
+
+	ref.Components = p.extractComponents(doc, docURL)
+	ref.Composables = p.extractComposables(doc, docURL)
+	ref.Utilities = p.extractUtilities(doc, docURL)
+	ref.Configs = p.extractConfigs(doc, docURL)
+	ref.Commands = p.extractCommands(doc, docURL)
+
+	return ref, nil
+}
+
+func (p *Parser) ParseSearchResults(html string) ([]*SearchResult, error) {
+	doc, err := goquery.NewDocumentFromReader(strings.NewReader(html))
+	if err != nil {
+		return nil, err
+	}
+
+	var results []*SearchResult
+
+	doc.Find(".search-result, a[href*='/docs/'], a[href*='/api/'], .nav-link").Each(func(i int, s *goquery.Selection) {
+		result := &SearchResult{}
+
+		result.Name = strings.TrimSpace(s.Text())
+
+		if href, exists := s.Attr("href"); exists {
+			result.DocURL = resolveURL(p.baseURL, href)
+
+			if strings.Contains(href, "/components/") {
+				result.Kind = "component"
+			} else if strings.Contains(href, "/composables/") {
+				result.Kind = "composable"
+			} else if strings.Contains(href, "/utils/") {
+				result.Kind = "utility"
+			} else if strings.Contains(href, "/config/") || strings.Contains(href, "/configuration/") {
+				result.Kind = "config"
+			} else if strings.Contains(href, "/commands/") {
+				result.Kind = "command"
+			} else {
+				result.Kind = "doc"
+			}
+		}
+
+		if result.Name != "" {
+			results = append(results, result)
+		}
+	})
+
+	return results, nil
+}
+
+func (p *Parser) extractComponents(doc *goquery.Document, docURL string) []*Component {
+	var components []*Component
+
+	nuxtComponents := []string{"NuxtPage", "NuxtLayout", "NuxtLink", "NuxtLoadingIndicator", "NuxtErrorBoundary", "NuxtPicture", "NuxtImg", "ClientOnly", "DevOnly"}
+
+	doc.Find("h1, h2, h3, .api-item, [id]").Each(func(_ int, s *goquery.Selection) {
+		id, _ := s.Attr("id")
+		text := s.Text()
+
+		for _, name := range nuxtComponents {
+			if id == name || strings.Contains(text, name) || strings.Contains(text, "<"+name) {
+				comp := &Component{
+					Name: name,
+				}
+
+				comp.DocURL = docURL + "#" + name
+
+				next := s.Next()
+				for next.Length() > 0 && !next.Is("h1, h2, h3") {
+					if next.Is("p") && comp.Doc == "" {
+						comp.Doc = strings.TrimSpace(next.Text())
+					}
+					next = next.Next()
+				}
+
+				components = append(components, comp)
+				break
+			}
+		}
+	})
+
+	return components
+}
+
+func (p *Parser) extractComposables(doc *goquery.Document, docURL string) []*Composable {
+	var composables []*Composable
+
+	nuxtComposables := []string{"useAsyncData", "useFetch", "useLazyAsyncData", "useLazyFetch", "useNuxtData", "useHead", "useHeadSafe", "useSeoMeta", "useRoute", "useRouter", "useState", "useCookie", "useRequestURL", "useRequestEvent", "useRequestHeaders", "useResponseHeader", "useRuntimeConfig", "useAppConfig", "useError", "createError", "isNuxtError", "showError", "throwError", "clearError", "reloadNuxtApp", "useRequestFetch", "useHydration", "usePreviewMode", "onPrehydrate"}
+
+	doc.Find("h1, h2, h3, .api-item, code").Each(func(_ int, s *goquery.Selection) {
+		id, _ := s.Attr("id")
+		text := s.Text()
+
+		for _, name := range nuxtComposables {
+			if id == name || strings.Contains(text, name+"(") {
+				comp := &Composable{
+					Name: name,
+				}
+
+				comp.DocURL = docURL + "#" + name
+
+				next := s.Next()
+				for next.Length() > 0 && !next.Is("h1, h2, h3") {
+					if next.Is("p") && comp.Doc == "" {
+						comp.Doc = strings.TrimSpace(next.Text())
+					}
+					next = next.Next()
+				}
+
+				composables = append(composables, comp)
+				break
+			}
+		}
+	})
+
+	return composables
+}
+
+func (p *Parser) extractUtilities(doc *goquery.Document, docURL string) []*Utility {
+	var utilities []*Utility
+
+	nuxtUtils := []string{"navigateTo", "abortNavigation", "setPageLayout", "defineNuxtComponent", "defineNuxtPlugin", "definePayloadHandler", "defineNuxtRouteMiddleware", "definePageMeta", "defineNuxtModule", "addComponent", "addImports", "addPluginTemplate", "createResolver"}
+
+	doc.Find("h1, h2, h3, .api-item, code").Each(func(_ int, s *goquery.Selection) {
+		id, _ := s.Attr("id")
+		text := s.Text()
+
+		for _, name := range nuxtUtils {
+			if id == name || strings.Contains(text, name+"(") {
+				util := &Utility{
+					Name: name,
+				}
+
+				util.DocURL = docURL + "#" + name
+
+				next := s.Next()
+				for next.Length() > 0 && !next.Is("h1, h2, h3") {
+					if next.Is("p") && util.Doc == "" {
+						util.Doc = strings.TrimSpace(next.Text())
+					}
+					next = next.Next()
+				}
+
+				utilities = append(utilities, util)
+				break
+			}
+		}
+	})
+
+	return utilities
+}
+
+func (p *Parser) extractConfigs(doc *goquery.Document, docURL string) []*Config {
+	var configs []*Config
+
+	nuxtConfigs := []string{"app", "build", "builder", "components", "compatibilityDate", "content", "css", "devtools", "extends", "experimental", "features", "generate", "hooks", "ignore", "imports", "logLevel", "modules", "nitro", "optimization", "pages", "plugins", "postcss", "prepare", "rootDir", "runtimeConfig", "serverDir", "sourcemap", "srcDir", "ssr", "telemetry", "testUtils", "typescript", "vite", "vue", "watchers", "workspaceDir"}
+
+	doc.Find("h1, h2, h3, .api-item, [id]").Each(func(_ int, s *goquery.Selection) {
+		id, _ := s.Attr("id")
+		text := s.Text()
+
+		for _, name := range nuxtConfigs {
+			if id == name || strings.Contains(strings.ToLower(text), name) {
+				cfg := &Config{
+					Name: name,
+				}
+
+				cfg.DocURL = docURL + "#" + name
+
+				next := s.Next()
+				for next.Length() > 0 && !next.Is("h1, h2, h3") {
+					if next.Is("p") && cfg.Doc == "" {
+						cfg.Doc = strings.TrimSpace(next.Text())
+					}
+					next = next.Next()
+				}
+
+				configs = append(configs, cfg)
+				break
+			}
+		}
+	})
+
+	return configs
+}
+
+func (p *Parser) extractCommands(doc *goquery.Document, docURL string) []*Command {
+	var commands []*Command
+
+	nuxtCommands := []string{"nuxi dev", "nuxi build", "nuxi generate", "nuxi preview", "nuxi analyze", "nuxi cleanup", "nuxi typecheck", "nuxi module", "nuxi info", "nuxi prepare", "nuxi upgrade"}
+
+	doc.Find("h1, h2, h3, .api-item, code, pre").Each(func(_ int, s *goquery.Selection) {
+		id, _ := s.Attr("id")
+		text := s.Text()
+
+		for _, name := range nuxtCommands {
+			if id == name || strings.Contains(text, name) {
+				cmd := &Command{
+					Name: name,
+				}
+
+				cmd.DocURL = docURL + "#" + strings.ReplaceAll(name, " ", "-")
+
+				next := s.Next()
+				for next.Length() > 0 && !next.Is("h1, h2, h3") {
+					if next.Is("p") && cmd.Doc == "" {
+						cmd.Doc = strings.TrimSpace(next.Text())
+					}
+					next = next.Next()
+				}
+
+				commands = append(commands, cmd)
+				break
+			}
+		}
+	})
+
+	return commands
+}
+
+func resolveURL(base string, href string) string {
+	if strings.HasPrefix(href, "http") {
+		return href
+	}
+
+	baseURL, err := url.Parse(base)
+	if err != nil {
+		return href
+	}
+
+	hrefURL, err := url.Parse(href)
+	if err != nil {
+		return href
+	}
+
+	return baseURL.ResolveReference(hrefURL).String()
+}

pkg/nuxtdocs/parser_test.go

@@ -0,0 +1,126 @@
+package nuxtdocs
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/PuerkitoBio/goquery"
+)
+
+const testReferencePageHTML = `
+<!DOCTYPE html>
+<html>
+<body>
+<h1>Nuxt API Reference</h1>
+
+<h2 id="useFetch">useFetch</h2>
+<p>Fetches data from an API endpoint with SSR support.</p>
+<pre><code>const { data, pending, error } = await useFetch('/api/data')</code></pre>
+
+<h2 id="useState">useState</h2>
+<p>Creates a reactive state that is shared across components.</p>
+
+<h2 id="NuxtPage">NuxtPage</h2>
+<p>Renders the current page component based on the route.</p>
+
+<h2 id="server">server</h2>
+<p>Nuxt server configuration options.</p>
+
+<h3 id="nuxi-dev">nuxi dev</h3>
+<p>Starts the development server.</p>
+</body>
+</html>
+`
+
+func TestParseReferencePage(t *testing.T) {
+	parser := NewParser()
+	ref, err := parser.ParseReferencePage(testReferencePageHTML, "https://nuxt.com/docs/api/")
+	if err != nil {
+		t.Fatalf("ParseReferencePage failed: %v", err)
+	}
+
+	if len(ref.Composables) == 0 {
+		t.Error("Expected at least one composable")
+	}
+
+	if len(ref.Components) == 0 {
+		t.Error("Expected at least one component")
+	}
+
+	if len(ref.Commands) == 0 {
+		t.Error("Expected at least one command")
+	}
+}
+
+func TestExtractComposables(t *testing.T) {
+	parser := NewParser()
+
+	doc, err := goquery.NewDocumentFromReader(strings.NewReader(testReferencePageHTML))
+	if err != nil {
+		t.Fatalf("Failed to parse HTML: %v", err)
+	}
+
+	composables := parser.extractComposables(doc, "https://nuxt.com/docs/api/")
+
+	if len(composables) == 0 {
+		t.Fatal("Expected at least one composable")
+	}
+
+	found := false
+	for _, c := range composables {
+		if c.Name == "useFetch" {
+			found = true
+			if c.Doc == "" {
+				t.Error("Expected useFetch to have documentation")
+			}
+			break
+		}
+	}
+
+	if !found {
+		t.Error("Expected to find useFetch composable")
+	}
+}
+
+func TestExtractComponents(t *testing.T) {
+	parser := NewParser()
+
+	doc, err := goquery.NewDocumentFromReader(strings.NewReader(testReferencePageHTML))
+	if err != nil {
+		t.Fatalf("Failed to parse HTML: %v", err)
+	}
+
+	components := parser.extractComponents(doc, "https://nuxt.com/docs/api/")
+
+	found := false
+	for _, c := range components {
+		if c.Name == "NuxtPage" {
+			found = true
+			break
+		}
+	}
+
+	if !found {
+		t.Error("Expected to find NuxtPage component")
+	}
+}
+
+func TestResolveURL(t *testing.T) {
+	tests := []struct {
+		base     string
+		href     string
+		expected string
+	}{
+		{"https://nuxt.com", "/docs/api/", "https://nuxt.com/docs/api/"},
+		{"https://nuxt.com", "https://example.com/page", "https://example.com/page"},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.href, func(t *testing.T) {
+			got := resolveURL(tt.base, tt.href)
+			if got != tt.expected {
+				t.Errorf("resolveURL(%q, %q) = %q, want %q", tt.base, tt.href, got, tt.expected)
+			}
+		})
+	}
+}

pkg/nuxtdocs/types.go

@@ -0,0 +1,159 @@
+// Package nuxtdocs provides parsing and extraction for Nuxt documentation
+// from nuxt.com/docs/.
+package nuxtdocs
+
+import "time"
+
+// Reference represents the Nuxt API reference.
+type Reference struct {
+	Version     string        `json:"version"`
+	DocURL      string        `json:"doc_url"`
+	Components  []*Component  `json:"components,omitempty"`
+	Composables []*Composable `json:"composables,omitempty"`
+	Utilities   []*Utility    `json:"utilities,omitempty"`
+	Configs     []*Config     `json:"configs,omitempty"`
+	Commands    []*Command    `json:"commands,omitempty"`
+	FetchedAt   time.Time     `json:"fetched_at"`
+}
+
+// Component represents a Nuxt component.
+type Component struct {
+	Name       string     `json:"name"`
+	Doc        string     `json:"doc,omitempty"`
+	Props      []*Prop    `json:"props,omitempty"`
+	Slots      []*Slot    `json:"slots,omitempty"`
+	Events     []*Event   `json:"events,omitempty"`
+	Examples   []*Example `json:"examples,omitempty"`
+	Category   string     `json:"category,omitempty"`
+	DocURL     string     `json:"doc_url"`
+	Deprecated string     `json:"deprecated,omitempty"`
+}
+
+// Prop represents a component prop.
+type Prop struct {
+	Name       string `json:"name"`
+	Type       string `json:"type,omitempty"`
+	Doc        string `json:"doc,omitempty"`
+	Required   bool   `json:"required"`
+	Default    string `json:"default,omitempty"`
+	Deprecated string `json:"deprecated,omitempty"`
+}
+
+// Slot represents a component slot.
+type Slot struct {
+	Name  string `json:"name"`
+	Doc   string `json:"doc,omitempty"`
+	Props string `json:"props,omitempty"`
+}
+
+// Event represents a component event.
+type Event struct {
+	Name       string       `json:"name"`
+	Doc        string       `json:"doc,omitempty"`
+	Parameters []*Parameter `json:"parameters,omitempty"`
+}
+
+// Composable represents a Nuxt composable.
+type Composable struct {
+	Name       string       `json:"name"`
+	Doc        string       `json:"doc,omitempty"`
+	Signature  string       `json:"signature,omitempty"`
+	Parameters []*Parameter `json:"parameters,omitempty"`
+	Returns    string       `json:"returns,omitempty"`
+	Examples   []*Example   `json:"examples,omitempty"`
+	Category   string       `json:"category,omitempty"` // data, state, routing, etc.
+	DocURL     string       `json:"doc_url"`
+	Deprecated string       `json:"deprecated,omitempty"`
+}
+
+// Utility represents a Nuxt utility function.
+type Utility struct {
+	Name       string       `json:"name"`
+	Doc        string       `json:"doc,omitempty"`
+	Signature  string       `json:"signature,omitempty"`
+	Parameters []*Parameter `json:"parameters,omitempty"`
+	Returns    string       `json:"returns,omitempty"`
+	Examples   []*Example   `json:"examples,omitempty"`
+	DocURL     string       `json:"doc_url"`
+	Deprecated string       `json:"deprecated,omitempty"`
+}
+
+// Config represents a Nuxt configuration option.
+type Config struct {
+	Name       string      `json:"name"`
+	Doc        string      `json:"doc,omitempty"`
+	Type       string      `json:"type,omitempty"`
+	Default    string      `json:"default,omitempty"`
+	Properties []*Property `json:"properties,omitempty"`
+	Examples   []*Example  `json:"examples,omitempty"`
+	Category   string      `json:"category,omitempty"`
+	DocURL     string      `json:"doc_url"`
+	Deprecated string      `json:"deprecated,omitempty"`
+}
+
+// Property represents a nested config property.
+type Property struct {
+	Name       string      `json:"name"`
+	Doc        string      `json:"doc,omitempty"`
+	Type       string      `json:"type,omitempty"`
+	Default    string      `json:"default,omitempty"`
+	Properties []*Property `json:"properties,omitempty"`
+	Deprecated string      `json:"deprecated,omitempty"`
+}
+
+// Command represents a Nuxt CLI command.
+type Command struct {
+	Name      string      `json:"name"`
+	Doc       string      `json:"doc,omitempty"`
+	Usage     string      `json:"usage,omitempty"`
+	Flags     []*Flag     `json:"flags,omitempty"`
+	Arguments []*Argument `json:"arguments,omitempty"`
+	Examples  []*Example  `json:"examples,omitempty"`
+	DocURL    string      `json:"doc_url"`
+}
+
+// Flag represents a command flag.
+type Flag struct {
+	Name     string `json:"name"`
+	Doc      string `json:"doc,omitempty"`
+	Type     string `json:"type,omitempty"`
+	Default  string `json:"default,omitempty"`
+	Required bool   `json:"required"`
+	Short    string `json:"short,omitempty"`
+}
+
+// Argument represents a command argument.
+type Argument struct {
+	Name     string `json:"name"`
+	Doc      string `json:"doc,omitempty"`
+	Type     string `json:"type,omitempty"`
+	Required bool   `json:"required"`
+	Default  string `json:"default,omitempty"`
+}
+
+// Parameter represents a function parameter.
+type Parameter struct {
+	Name     string `json:"name"`
+	Type     string `json:"type,omitempty"`
+	Doc      string `json:"doc,omitempty"`
+	Optional bool   `json:"optional"`
+	Default  string `json:"default,omitempty"`
+}
+
+// Example represents a code example.
+type Example struct {
+	Title       string `json:"title,omitempty"`
+	Code        string `json:"code"`
+	Description string `json:"description,omitempty"`
+	Language    string `json:"language,omitempty"`
+}
+
+// SearchResult represents a search result.
+type SearchResult struct {
+	Name       string `json:"name"`
+	Kind       string `json:"kind"` // component, composable, utility, config, command
+	Doc        string `json:"doc,omitempty"`
+	DocURL     string `json:"doc_url"`
+	Score      int    `json:"score"`
+	Deprecated bool   `json:"deprecated"`
+}

pkg/pythondocs/parser.go

@@ -0,0 +1,555 @@
+package pythondocs
+
+import (
+	"net/url"
+	"regexp"
+	"strings"
+	"time"
+
+	"github.com/PuerkitoBio/goquery"
+)
+
+type Parser struct {
+	baseURL string
+}
+
+func NewParser() *Parser {
+	return &Parser{
+		baseURL: "https://docs.python.org",
+	}
+}
+
+func (p *Parser) ParseModulePage(html string, docURL string) (*Module, error) {
+	doc, err := goquery.NewDocumentFromReader(strings.NewReader(html))
+	if err != nil {
+		return nil, err
+	}
+
+	module := &Module{
+		DocURL:    docURL,
+		FetchedAt: time.Now(),
+	}
+
+	module.Name = p.extractModuleName(doc)
+	module.Path = module.Name
+	module.Doc = p.extractModuleDoc(doc)
+	module.Synopsis = p.extractSynopsis(doc)
+	module.Version = p.extractVersion(doc)
+
+	module.Classes = p.extractClasses(doc, module.Name, docURL)
+	module.Functions = p.extractFunctions(doc, module.Name, docURL)
+	module.Exceptions = p.extractExceptions(doc, module.Name, docURL)
+	module.Constants = p.extractData(doc, module.Name, docURL)
+
+	return module, nil
+}
+
+func (p *Parser) ParseSearchResults(html string) ([]*SearchResult, error) {
+	doc, err := goquery.NewDocumentFromReader(strings.NewReader(html))
+	if err != nil {
+		return nil, err
+	}
+
+	var results []*SearchResult
+
+	doc.Find("ul.search li").Each(func(i int, s *goquery.Selection) {
+		result := &SearchResult{}
+
+		classes, _ := s.Attr("class")
+		if strings.Contains(classes, "kind-object") {
+			result.Kind = "object"
+		} else if strings.Contains(classes, "kind-text") {
+			result.Kind = "text"
+		} else if strings.Contains(classes, "kind-title") {
+			result.Kind = "title"
+		}
+
+		link := s.Find("a").First()
+		result.Name = strings.TrimSpace(link.Text())
+
+		if href, exists := link.Attr("href"); exists {
+			result.DocURL = resolveURL(p.baseURL, href)
+			result.Path = extractPathFromURL(href)
+		}
+
+		if score, exists := link.Attr("data-score"); exists {
+			var scoreInt int
+			for _, c := range score {
+				if c >= '0' && c <= '9' {
+					scoreInt = scoreInt*10 + int(c-'0')
+				}
+			}
+			result.Score = scoreInt
+		}
+
+		span := s.Find("span").Last()
+		result.Description = strings.TrimSpace(span.Text())
+
+		results = append(results, result)
+	})
+
+	return results, nil
+}
+
+func (p *Parser) extractModuleName(doc *goquery.Document) string {
+	section := doc.Find("section[id^='module-']").First()
+	if section.Length() > 0 {
+		id, _ := section.Attr("id")
+		return strings.TrimPrefix(id, "module-")
+	}
+
+	h1 := doc.Find("h1 code").First()
+	if h1.Length() > 0 {
+		return strings.TrimSpace(h1.Text())
+	}
+
+	h1 = doc.Find(".body h1").First()
+	if h1.Length() > 0 {
+		text := h1.Text()
+		if strings.HasPrefix(text, "—") {
+			parts := strings.SplitN(text, "—", 2)
+			if len(parts) > 0 {
+				return strings.TrimSpace(parts[0])
+			}
+		}
+		return strings.TrimSpace(text)
+	}
+
+	return ""
+}
+
+func (p *Parser) extractModuleDoc(doc *goquery.Document) string {
+	section := doc.Find("section[id^='module-']").First()
+	if section.Length() == 0 {
+		section = doc.Find(".body").First()
+	}
+
+	docblock := section.Find("p").First()
+	if docblock.Length() > 0 {
+		return strings.TrimSpace(docblock.Text())
+	}
+
+	return ""
+}
+
+func (p *Parser) extractSynopsis(doc *goquery.Document) string {
+	text := doc.Find(".body p").First().Text()
+	text = strings.TrimSpace(text)
+	if len(text) > 200 {
+		return text[:197] + "..."
+	}
+	return text
+}
+
+func (p *Parser) extractVersion(doc *goquery.Document) string {
+	versionAdded := doc.Find(".versionadded").Text()
+	if versionAdded != "" {
+		re := regexp.MustCompile(`\d+\.\d+`)
+		if match := re.FindString(versionAdded); match != "" {
+			return match
+		}
+	}
+
+	versionChanged := doc.Find(".versionchanged").Text()
+	if versionChanged != "" {
+		re := regexp.MustCompile(`\d+\.\d+`)
+		if match := re.FindString(versionChanged); match != "" {
+			return match
+		}
+	}
+
+	return ""
+}
+
+func (p *Parser) extractClasses(doc *goquery.Document, moduleName string, docURL string) []*Class {
+	var classes []*Class
+
+	doc.Find("dl.py.class").Each(func(_ int, s *goquery.Selection) {
+		class := &Class{
+			Module: moduleName,
+		}
+
+		dt := s.Find("dt.sig-object").First()
+		if dt.Length() == 0 {
+			dt = s.Find("dt").First()
+		}
+
+		sig := dt.Find("code.sig-prename")
+		class.Name = strings.TrimSpace(sig.Find(".pre").Last().Text())
+		if class.Name == "" {
+			class.Name = strings.TrimSpace(dt.Find(".sig-name").Text())
+		}
+		if class.Name == "" {
+			sigText := dt.Text()
+			sigText = strings.TrimSpace(sigText)
+			parts := strings.Fields(sigText)
+			if len(parts) > 0 {
+				class.Name = parts[0]
+			}
+		}
+
+		if id, exists := dt.Attr("id"); exists {
+			class.QualName = id
+			class.DocURL = docURL + "#" + id
+		} else {
+			class.QualName = class.Name
+			class.DocURL = docURL
+		}
+
+		class.Signature = strings.TrimSpace(dt.Text())
+
+		dd := s.Find("dd").First()
+		class.Doc = strings.TrimSpace(dd.Find("p").First().Text())
+
+		bases := dt.Find("a.reference.internal")
+		bases.Each(func(_ int, b *goquery.Selection) {
+			base := strings.TrimSpace(b.Text())
+			if base != "" && base != class.Name {
+				class.Bases = append(class.Bases, base)
+			}
+		})
+
+		class.Methods = p.extractMethods(s, class.Name, docURL)
+		class.ClassMethods = p.extractClassMethods(s, class.Name, docURL)
+		class.StaticMethods = p.extractStaticMethods(s, class.Name, docURL)
+		class.Attributes = p.extractAttributes(s, class.Name, docURL)
+
+		if class.Name != "" {
+			classes = append(classes, class)
+		}
+	})
+
+	return classes
+}
+
+func (p *Parser) extractFunctions(doc *goquery.Document, moduleName string, docURL string) []*Function {
+	var functions []*Function
+
+	doc.Find("dl.py.function").Each(func(_ int, s *goquery.Selection) {
+		fn := &Function{
+			Module: moduleName,
+		}
+
+		dt := s.Find("dt.sig-object").First()
+		if dt.Length() == 0 {
+			dt = s.Find("dt").First()
+		}
+
+		sig := dt.Find("code.sig-prename")
+		fn.Name = strings.TrimSpace(sig.Find(".pre").Last().Text())
+		if fn.Name == "" {
+			fn.Name = strings.TrimSpace(dt.Find(".sig-name").Text())
+		}
+		if fn.Name == "" {
+			sigText := dt.Text()
+			sigText = strings.TrimSpace(sigText)
+			if idx := strings.Index(sigText, "("); idx > 0 {
+				fn.Name = strings.TrimSpace(sigText[:idx])
+			}
+		}
+
+		if id, exists := dt.Attr("id"); exists {
+			fn.QualName = id
+			fn.DocURL = docURL + "#" + id
+		} else {
+			fn.QualName = fn.Name
+			fn.DocURL = docURL
+		}
+
+		fn.Signature = strings.TrimSpace(dt.Text())
+
+		dd := s.Find("dd").First()
+		fn.Doc = strings.TrimSpace(dd.Find("p").First().Text())
+
+		fn.Parameters = p.extractParameters(dt)
+
+		if class := s.Find("dl.py.method, dl.py.classmethod, dl.py.staticmethod"); class.Length() > 0 {
+			return
+		}
+
+		if fn.Name != "" {
+			functions = append(functions, fn)
+		}
+	})
+
+	return functions
+}
+
+func (p *Parser) extractExceptions(doc *goquery.Document, moduleName string, docURL string) []*Exception {
+	var exceptions []*Exception
+
+	doc.Find("dl.py.exception").Each(func(_ int, s *goquery.Selection) {
+		exc := &Exception{
+			Module: moduleName,
+		}
+
+		dt := s.Find("dt.sig-object").First()
+		if dt.Length() == 0 {
+			dt = s.Find("dt").First()
+		}
+
+		sig := dt.Find("code.sig-prename")
+		exc.Name = strings.TrimSpace(sig.Find(".pre").Last().Text())
+		if exc.Name == "" {
+			exc.Name = strings.TrimSpace(dt.Find(".sig-name").Text())
+		}
+		if exc.Name == "" {
+			sigText := dt.Text()
+			sigText = strings.TrimSpace(sigText)
+			if idx := strings.Index(sigText, "("); idx > 0 {
+				exc.Name = strings.TrimSpace(sigText[:idx])
+			}
+		}
+
+		if id, exists := dt.Attr("id"); exists {
+			exc.QualName = id
+			exc.DocURL = docURL + "#" + id
+		} else {
+			exc.QualName = exc.Name
+			exc.DocURL = docURL
+		}
+
+		exc.Signature = strings.TrimSpace(dt.Text())
+
+		dd := s.Find("dd").First()
+		exc.Doc = strings.TrimSpace(dd.Find("p").First().Text())
+
+		if exc.Name != "" {
+			exceptions = append(exceptions, exc)
+		}
+	})
+
+	return exceptions
+}
+
+func (p *Parser) extractData(doc *goquery.Document, moduleName string, docURL string) []*Data {
+	var dataList []*Data
+
+	doc.Find("dl.py.data").Each(func(_ int, s *goquery.Selection) {
+		data := &Data{
+			Module: moduleName,
+		}
+
+		dt := s.Find("dt.sig-object").First()
+		if dt.Length() == 0 {
+			dt = s.Find("dt").First()
+		}
+
+		sig := dt.Find("code.sig-prename")
+		data.Name = strings.TrimSpace(sig.Find(".pre").Last().Text())
+		if data.Name == "" {
+			sigText := dt.Text()
+			sigText = strings.TrimSpace(sigText)
+			data.Name = strings.Fields(sigText)[0]
+		}
+
+		if id, exists := dt.Attr("id"); exists {
+			data.DocURL = docURL + "#" + id
+		} else {
+			data.DocURL = docURL
+		}
+
+		dd := s.Find("dd").First()
+		data.Doc = strings.TrimSpace(dd.Find("p").First().Text())
+
+		if data.Name != "" {
+			dataList = append(dataList, data)
+		}
+	})
+
+	return dataList
+}
+
+func (p *Parser) extractMethods(parent *goquery.Selection, className string, docURL string) []*Method {
+	var methods []*Method
+
+	parent.Find("dl.py.method").Each(func(_ int, s *goquery.Selection) {
+		method := p.parseMethod(s, className, docURL, false, false)
+		if method != nil {
+			methods = append(methods, method)
+		}
+	})
+
+	return methods
+}
+
+func (p *Parser) extractClassMethods(parent *goquery.Selection, className string, docURL string) []*Method {
+	var methods []*Method
+
+	parent.Find("dl.py.classmethod").Each(func(_ int, s *goquery.Selection) {
+		method := p.parseMethod(s, className, docURL, true, false)
+		if method != nil {
+			methods = append(methods, method)
+		}
+	})
+
+	return methods
+}
+
+func (p *Parser) extractStaticMethods(parent *goquery.Selection, className string, docURL string) []*Method {
+	var methods []*Method
+
+	parent.Find("dl.py.staticmethod").Each(func(_ int, s *goquery.Selection) {
+		method := p.parseMethod(s, className, docURL, false, true)
+		if method != nil {
+			methods = append(methods, method)
+		}
+	})
+
+	return methods
+}
+
+func (p *Parser) parseMethod(s *goquery.Selection, className string, docURL string, isClassMethod bool, isStatic bool) *Method {
+	method := &Method{
+		Class:         className,
+		IsClassMethod: isClassMethod,
+		IsStatic:      isStatic,
+	}
+
+	dt := s.Find("dt.sig-object").First()
+	if dt.Length() == 0 {
+		dt = s.Find("dt").First()
+	}
+
+	sig := dt.Find("code.sig-prename")
+	method.Name = strings.TrimSpace(sig.Find(".pre").Last().Text())
+	if method.Name == "" {
+		method.Name = strings.TrimSpace(dt.Find(".sig-name").Text())
+	}
+	if method.Name == "" {
+		sigText := dt.Text()
+		sigText = strings.TrimSpace(sigText)
+		if idx := strings.Index(sigText, "("); idx > 0 {
+			name := strings.TrimSpace(sigText[:idx])
+			parts := strings.Split(name, ".")
+			method.Name = parts[len(parts)-1]
+		}
+	}
+
+	if id, exists := dt.Attr("id"); exists {
+		method.QualName = id
+		method.DocURL = docURL + "#" + id
+	} else {
+		method.QualName = className + "." + method.Name
+		method.DocURL = docURL
+	}
+
+	method.Signature = strings.TrimSpace(dt.Text())
+
+	dd := s.Find("dd").First()
+	method.Doc = strings.TrimSpace(dd.Find("p").First().Text())
+
+	method.Parameters = p.extractParameters(dt)
+
+	if method.Name != "" {
+		return method
+	}
+	return nil
+}
+
+func (p *Parser) extractAttributes(parent *goquery.Selection, className string, docURL string) []*Attribute {
+	var attributes []*Attribute
+
+	parent.Find("dl.py.attribute").Each(func(_ int, s *goquery.Selection) {
+		attr := &Attribute{
+			Class: className,
+		}
+
+		dt := s.Find("dt.sig-object").First()
+		if dt.Length() == 0 {
+			dt = s.Find("dt").First()
+		}
+
+		sig := dt.Find("code.sig-prename")
+		attr.Name = strings.TrimSpace(sig.Find(".pre").Last().Text())
+		if attr.Name == "" {
+			sigText := dt.Text()
+			sigText = strings.TrimSpace(sigText)
+			attr.Name = strings.Fields(sigText)[0]
+		}
+
+		if id, exists := dt.Attr("id"); exists {
+			attr.DocURL = docURL + "#" + id
+		} else {
+			attr.DocURL = docURL
+		}
+
+		dd := s.Find("dd").First()
+		attr.Doc = strings.TrimSpace(dd.Find("p").First().Text())
+
+		if attr.Name != "" {
+			attributes = append(attributes, attr)
+		}
+	})
+
+	return attributes
+}
+
+func (p *Parser) extractParameters(dt *goquery.Selection) []*Param {
+	var params []*Param
+
+	dt.Find("em.sig-param").Each(func(_ int, em *goquery.Selection) {
+		param := &Param{}
+
+		text := strings.TrimSpace(em.Text())
+
+		if strings.HasPrefix(text, "*") && !strings.HasPrefix(text, "**") {
+			param.IsVarArgs = true
+			text = strings.TrimPrefix(text, "*")
+		} else if strings.HasPrefix(text, "**") {
+			param.IsKWArgs = true
+			text = strings.TrimPrefix(text, "**")
+		}
+
+		if strings.Contains(text, "=") {
+			parts := strings.SplitN(text, "=", 2)
+			param.Name = strings.TrimSpace(parts[0])
+			param.Default = strings.TrimSpace(parts[1])
+		} else {
+			param.Name = text
+		}
+
+		if param.Name != "" {
+			params = append(params, param)
+		}
+	})
+
+	return params
+}
+
+func extractPathFromURL(href string) string {
+	u, err := url.Parse(href)
+	if err != nil {
+		return href
+	}
+
+	path := u.Path
+	path = strings.TrimSuffix(path, ".html")
+	path = strings.TrimSuffix(path, "/")
+	path = strings.TrimPrefix(path, "/")
+
+	if strings.Contains(path, "#") {
+		parts := strings.Split(path, "#")
+		path = parts[0]
+	}
+
+	return path
+}
+
+func resolveURL(base string, href string) string {
+	if strings.HasPrefix(href, "http") {
+		return href
+	}
+
+	baseURL, err := url.Parse(base)
+	if err != nil {
+		return href
+	}
+
+	hrefURL, err := url.Parse(href)
+	if err != nil {
+		return href
+	}
+
+	return baseURL.ResolveReference(hrefURL).String()
+}

pkg/pythondocs/parser_test.go

@@ -0,0 +1,281 @@
+package pythondocs
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/PuerkitoBio/goquery"
+)
+
+const testModulePageHTML = `
+<!DOCTYPE html>
+<html>
+<body>
+<div class="body" role="main">
+<section id="module-test">
+<h1><code class="xref py py-mod docutils literal notranslate"><span class="pre">test</span></code> — Regression tests package<a class="headerlink" href="#module-test">¶</a></h1>
+<p>The test package contains all regression tests for Python.</p>
+<p>This is additional documentation.</p>
+
+<dl class="py class">
+<dt class="sig sig-object py" id="test.TestCase">
+<em class="property"><span class="pre">class</span></em>
+<span class="sig-prename descclassname"><span class="pre">test.</span></span>
+<span class="sig-name descname"><span class="pre">TestCase</span></span>
+<span class="sig-paren">(</span><em class="sig-param"><span class="pre">methodName</span><span class="pre">=</span><span class="pre">'runTest'</span></em><span class="sig-paren">)</span>
+<a class="headerlink" href="#test.TestCase">¶</a>
+</dt>
+<dd><p>A test case class.</p></dd>
+</dl>
+
+<dl class="py function">
+<dt class="sig sig-object py" id="test.run_test">
+<span class="sig-prename descclassname"><span class="pre">test.</span></span>
+<span class="sig-name descname"><span class="pre">run_test</span></span>
+<span class="sig-paren">(</span><em class="sig-param"><span class="pre">name</span></em>, <em class="sig-param"><span class="pre">verbose</span><span class="pre">=</span><span class="pre">False</span></em><span class="sig-paren">)</span>
+<a class="headerlink" href="#test.run_test">¶</a>
+</dt>
+<dd><p>Run a single test.</p></dd>
+</dl>
+
+<dl class="py exception">
+<dt class="sig sig-object py" id="test.TestFailed">
+<em class="property"><span class="pre">exception</span></em>
+<span class="sig-prename descclassname"><span class="pre">test.</span></span>
+<span class="sig-name descname"><span class="pre">TestFailed</span></span>
+<a class="headerlink" href="#test.TestFailed">¶</a>
+</dt>
+<dd><p>Exception raised when a test fails.</p></dd>
+</dl>
+
+<dl class="py data">
+<dt class="sig sig-object py" id="test.verbose">
+<span class="sig-prename descclassname"><span class="pre">test.</span></span>
+<span class="sig-name descname"><span class="pre">verbose</span></span>
+<a class="headerlink" href="#test.verbose">¶</a>
+</dt>
+<dd><p>True when verbose output is enabled.</p></dd>
+</dl>
+
+</section>
+</div>
+</body>
+</html>
+`
+
+const testClassHTML = `
+<dl class="py class">
+<dt class="sig sig-object py" id="test.TestCase">
+<em class="property"><span class="pre">class</span></em>
+<span class="sig-prename descclassname"><span class="pre">test.</span></span>
+<span class="sig-name descname"><span class="pre">TestCase</span></span>
+<span class="sig-paren">(</span><em class="sig-param"><span class="pre">methodName</span><span class="pre">=</span><span class="pre">'runTest'</span></em><span class="sig-paren">)</span>
+</dt>
+<dd>
+<p>A test case class that provides testing functionality.</p>
+
+<dl class="py method">
+<dt class="sig sig-object py" id="test.TestCase.setUp">
+<span class="sig-prename descclassname"><span class="pre">test.TestCase.</span></span>
+<span class="sig-name descname"><span class="pre">setUp</span></span>
+<span class="sig-paren">(</span><span class="sig-paren">)</span>
+</dt>
+<dd><p>Set up the test fixture.</p></dd>
+</dl>
+
+<dl class="py method">
+<dt class="sig sig-object py" id="test.TestCase.tearDown">
+<span class="sig-prename descclassname"><span class="pre">test.TestCase.</span></span>
+<span class="sig-name descname"><span class="pre">tearDown</span></span>
+<span class="sig-paren">(</span><span class="sig-paren">)</span>
+</dt>
+<dd><p>Tear down the test fixture.</p></dd>
+</dl>
+
+<dl class="py classmethod">
+<dt class="sig sig-object py" id="test.TestCase.setUpClass">
+<em class="property"><span class="pre">classmethod</span></em>
+<span class="sig-prename descclassname"><span class="pre">test.TestCase.</span></span>
+<span class="sig-name descname"><span class="pre">setUpClass</span></span>
+<span class="sig-paren">(</span><span class="sig-paren">)</span>
+</dt>
+<dd><p>Set up the test class.</p></dd>
+</dl>
+
+<dl class="py attribute">
+<dt class="sig sig-object py" id="test.TestCase.maxDiff">
+<span class="sig-prename descclassname"><span class="pre">test.TestCase.</span></span>
+<span class="sig-name descname"><span class="pre">maxDiff</span></span>
+</dt>
+<dd><p>Maximum diff length.</p></dd>
+</dl>
+
+</dd>
+</dl>
+`
+
+const testSearchHTML = `
+<ul class="search">
+<li class="kind-object">
+<a href="library/test.html#module-test" data-score="26">test</a>
+<span>(Python module, in test — Regression tests package)</span>
+</li>
+<li class="kind-object">
+<a href="library/unittest.html#module-unittest" data-score="21">unittest</a>
+<span>(Python module, in unittest — Unit testing framework)</span>
+</li>
+<li class="kind-text">
+<a href="library/keyword.html" data-score="15">keyword — Testing for Python keywords</a>
+<p class="context">This module allows a Python program to determine if a string is a keyword.</p>
+</li>
+</ul>
+`
+
+func TestParseModulePage(t *testing.T) {
+	parser := NewParser()
+	module, err := parser.ParseModulePage(testModulePageHTML, "https://docs.python.org/3/library/test.html")
+	if err != nil {
+		t.Fatalf("ParseModulePage failed: %v", err)
+	}
+
+	if module.Name == "" {
+		t.Error("Expected non-empty module name")
+	}
+
+	if module.Doc == "" {
+		t.Error("Expected non-empty doc")
+	}
+
+	if len(module.Classes) == 0 {
+		t.Error("Expected at least one class")
+	}
+
+	if len(module.Functions) == 0 {
+		t.Error("Expected at least one function")
+	}
+
+	if len(module.Exceptions) == 0 {
+		t.Error("Expected at least one exception")
+	}
+
+	if len(module.Constants) == 0 {
+		t.Error("Expected at least one constant/data")
+	}
+}
+
+func TestParseSearchResults(t *testing.T) {
+	parser := NewParser()
+	results, err := parser.ParseSearchResults(testSearchHTML)
+	if err != nil {
+		t.Fatalf("ParseSearchResults failed: %v", err)
+	}
+
+	if len(results) < 2 {
+		t.Fatalf("Expected at least 2 results, got %d", len(results))
+	}
+
+	first := results[0]
+	if first.Name == "" {
+		t.Error("Expected non-empty name")
+	}
+
+	if first.DocURL == "" {
+		t.Error("Expected non-empty doc URL")
+	}
+}
+
+func TestExtractClasses(t *testing.T) {
+	parser := NewParser()
+
+	doc, err := goquery.NewDocumentFromReader(strings.NewReader(testClassHTML))
+	if err != nil {
+		t.Fatalf("Failed to parse HTML: %v", err)
+	}
+
+	classes := parser.extractClasses(doc, "test", "https://docs.python.org/3/library/test.html")
+
+	if len(classes) == 0 {
+		t.Fatal("Expected at least one class")
+	}
+
+	tc := classes[0]
+	if tc.Name == "" {
+		t.Error("Expected non-empty class name")
+	}
+
+	if len(tc.Methods) < 2 {
+		t.Errorf("Expected at least 2 methods, got %d", len(tc.Methods))
+	}
+
+	if len(tc.ClassMethods) == 0 {
+		t.Error("Expected at least one classmethod")
+	}
+
+	if len(tc.Attributes) == 0 {
+		t.Error("Expected at least one attribute")
+	}
+}
+
+func TestExtractFunctions(t *testing.T) {
+	parser := NewParser()
+
+	doc, err := goquery.NewDocumentFromReader(strings.NewReader(testModulePageHTML))
+	if err != nil {
+		t.Fatalf("Failed to parse HTML: %v", err)
+	}
+
+	functions := parser.extractFunctions(doc, "test", "https://docs.python.org/3/library/test.html")
+
+	if len(functions) == 0 {
+		t.Fatal("Expected at least one function")
+	}
+
+	fn := functions[0]
+	if fn.Name == "" {
+		t.Error("Expected non-empty function name")
+	}
+
+	if fn.Signature == "" {
+		t.Error("Expected non-empty signature")
+	}
+}
+
+func TestResolveURL(t *testing.T) {
+	tests := []struct {
+		base     string
+		href     string
+		expected string
+	}{
+		{"https://docs.python.org", "/library/test.html", "https://docs.python.org/library/test.html"},
+		{"https://docs.python.org", "https://example.com/page", "https://example.com/page"},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.href, func(t *testing.T) {
+			got := resolveURL(tt.base, tt.href)
+			if got != tt.expected {
+				t.Errorf("resolveURL(%q, %q) = %q, want %q", tt.base, tt.href, got, tt.expected)
+			}
+		})
+	}
+}
+
+func TestExtractPathFromURL(t *testing.T) {
+	tests := []struct {
+		href     string
+		expected string
+	}{
+		{"/library/test.html", "library/test"},
+		{"library/test.html", "library/test"},
+		{"/library/test.html#module-test", "library/test"},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.href, func(t *testing.T) {
+			got := extractPathFromURL(tt.href)
+			if got != tt.expected {
+				t.Errorf("extractPathFromURL(%q) = %q, want %q", tt.href, got, tt.expected)
+			}
+		})
+	}
+}

pkg/pythondocs/types.go

@@ -0,0 +1,168 @@
+package pythondocs
+
+import "time"
+
+type Module struct {
+	Name       string       `json:"name"`
+	Path       string       `json:"path"`
+	Doc        string       `json:"doc,omitempty"`
+	Synopsis   string       `json:"synopsis,omitempty"`
+	Classes    []*Class     `json:"classes,omitempty"`
+	Functions  []*Function  `json:"functions,omitempty"`
+	Exceptions []*Exception `json:"exceptions,omitempty"`
+	Constants  []*Data      `json:"constants,omitempty"`
+	Submodules []string     `json:"submodules,omitempty"`
+	DocURL     string       `json:"doc_url"`
+	Version    string       `json:"version,omitempty"`
+	FetchedAt  time.Time    `json:"fetched_at"`
+}
+
+type Package struct {
+	Name        string    `json:"name"`
+	Version     string    `json:"version,omitempty"`
+	Description string    `json:"description,omitempty"`
+	Modules     []*Module `json:"modules,omitempty"`
+	DocURL      string    `json:"doc_url"`
+	Repository  string    `json:"repository,omitempty"`
+	License     string    `json:"license,omitempty"`
+	FetchedAt   time.Time `json:"fetched_at"`
+}
+
+type Class struct {
+	Name          string       `json:"name"`
+	Module        string       `json:"module"`
+	QualName      string       `json:"qual_name"`
+	Doc           string       `json:"doc,omitempty"`
+	Signature     string       `json:"signature,omitempty"`
+	Bases         []string     `json:"bases,omitempty"`
+	Methods       []*Method    `json:"methods,omitempty"`
+	ClassMethods  []*Method    `json:"classmethods,omitempty"`
+	StaticMethods []*Method    `json:"staticmethods,omitempty"`
+	Attributes    []*Attribute `json:"attributes,omitempty"`
+	Properties    []*Property  `json:"properties,omitempty"`
+	DocURL        string       `json:"doc_url"`
+}
+
+type Function struct {
+	Name        string   `json:"name"`
+	Module      string   `json:"module"`
+	QualName    string   `json:"qual_name"`
+	Doc         string   `json:"doc,omitempty"`
+	Signature   string   `json:"signature"`
+	Parameters  []*Param `json:"parameters,omitempty"`
+	Returns     *Return  `json:"returns,omitempty"`
+	Raises      []string `json:"raises,omitempty"`
+	Examples    []string `json:"examples,omitempty"`
+	DocURL      string   `json:"doc_url"`
+	IsAsync     bool     `json:"is_async"`
+	IsGenerator bool     `json:"is_generator"`
+	Decorator   string   `json:"decorator,omitempty"`
+}
+
+type Method struct {
+	Name          string   `json:"name"`
+	Class         string   `json:"class"`
+	QualName      string   `json:"qual_name"`
+	Doc           string   `json:"doc,omitempty"`
+	Signature     string   `json:"signature"`
+	Parameters    []*Param `json:"parameters,omitempty"`
+	Returns       *Return  `json:"returns,omitempty"`
+	Raises        []string `json:"raises,omitempty"`
+	Examples      []string `json:"examples,omitempty"`
+	DocURL        string   `json:"doc_url"`
+	IsAsync       bool     `json:"is_async"`
+	IsClassMethod bool     `json:"is_classmethod"`
+	IsStatic      bool     `json:"is_static"`
+	IsAbstract    bool     `json:"is_abstract"`
+	IsProperty    bool     `json:"is_property"`
+}
+
+type Property struct {
+	Name       string `json:"name"`
+	Class      string `json:"class"`
+	Doc        string `json:"doc,omitempty"`
+	Type       string `json:"type,omitempty"`
+	Getter     string `json:"getter,omitempty"`
+	Setter     string `json:"setter,omitempty"`
+	Deleter    string `json:"deleter,omitempty"`
+	DocURL     string `json:"doc_url"`
+	IsReadOnly bool   `json:"is_readonly"`
+}
+
+type Attribute struct {
+	Name   string `json:"name"`
+	Class  string `json:"class"`
+	Doc    string `json:"doc,omitempty"`
+	Type   string `json:"type,omitempty"`
+	Value  string `json:"value,omitempty"`
+	DocURL string `json:"doc_url"`
+}
+
+type Exception struct {
+	Name      string   `json:"name"`
+	Module    string   `json:"module"`
+	QualName  string   `json:"qual_name"`
+	Doc       string   `json:"doc,omitempty"`
+	Signature string   `json:"signature,omitempty"`
+	Bases     []string `json:"bases,omitempty"`
+	DocURL    string   `json:"doc_url"`
+}
+
+type Data struct {
+	Name    string `json:"name"`
+	Module  string `json:"module"`
+	Doc     string `json:"doc,omitempty"`
+	Type    string `json:"type,omitempty"`
+	Value   string `json:"value,omitempty"`
+	DocURL  string `json:"doc_url"`
+	IsConst bool   `json:"is_const"`
+}
+
+type Param struct {
+	Name         string `json:"name"`
+	Type         string `json:"type,omitempty"`
+	Default      string `json:"default,omitempty"`
+	Doc          string `json:"doc,omitempty"`
+	IsPositional bool   `json:"is_positional"`
+	IsKeyword    bool   `json:"is_keyword"`
+	IsVarArgs    bool   `json:"is_varargs"`
+	IsKWArgs     bool   `json:"is_kwargs"`
+}
+
+type Return struct {
+	Type string `json:"type,omitempty"`
+	Doc  string `json:"doc,omitempty"`
+}
+
+type SearchResult struct {
+	Name        string `json:"name"`
+	Kind        string `json:"kind"`
+	Path        string `json:"path"`
+	Module      string `json:"module,omitempty"`
+	Description string `json:"description,omitempty"`
+	DocURL      string `json:"doc_url"`
+	Score       int    `json:"score"`
+}
+
+type ItemKind string
+
+const (
+	ItemKindModule    ItemKind = "module"
+	ItemKindClass     ItemKind = "class"
+	ItemKindFunction  ItemKind = "function"
+	ItemKindMethod    ItemKind = "method"
+	ItemKindException ItemKind = "exception"
+	ItemKindData      ItemKind = "data"
+	ItemKindAttribute ItemKind = "attribute"
+	ItemKindProperty  ItemKind = "property"
+)
+
+type Symbol struct {
+	Name      string   `json:"name"`
+	Kind      ItemKind `json:"kind"`
+	Module    string   `json:"module"`
+	QualName  string   `json:"qual_name,omitempty"`
+	Signature string   `json:"signature,omitempty"`
+	Doc       string   `json:"doc,omitempty"`
+	DocURL    string   `json:"doc_url"`
+}

pkg/reactdocs/parser.go

@@ -0,0 +1,286 @@
+package reactdocs
+
+import (
+	"net/url"
+	"strings"
+	"time"
+
+	"github.com/PuerkitoBio/goquery"
+)
+
+type Parser struct {
+	baseURL string
+}
+
+func NewParser() *Parser {
+	return &Parser{
+		baseURL: "https://react.dev",
+	}
+}
+
+func (p *Parser) ParseReferencePage(html string, docURL string) (*Reference, error) {
+	doc, err := goquery.NewDocumentFromReader(strings.NewReader(html))
+	if err != nil {
+		return nil, err
+	}
+
+	ref := &Reference{
+		DocURL:    docURL,
+		FetchedAt: time.Now(),
+	}
+
+	ref.Hooks = p.extractHooks(doc, docURL)
+	ref.Components = p.extractComponents(doc, docURL)
+	ref.APIs = p.extractAPIs(doc, docURL)
+	ref.Directives = p.extractDirectives(doc, docURL)
+
+	return ref, nil
+}
+
+func (p *Parser) ParseSearchResults(html string) ([]*SearchResult, error) {
+	doc, err := goquery.NewDocumentFromReader(strings.NewReader(html))
+	if err != nil {
+		return nil, err
+	}
+
+	var results []*SearchResult
+
+	doc.Find(".search-result, a[href*='/reference/'], .nav-link").Each(func(i int, s *goquery.Selection) {
+		result := &SearchResult{}
+
+		result.Name = strings.TrimSpace(s.Text())
+
+		if href, exists := s.Attr("href"); exists {
+			result.DocURL = resolveURL(p.baseURL, href)
+
+			if strings.Contains(href, "/hooks/") {
+				result.Kind = "hook"
+			} else if strings.Contains(href, "/components/") {
+				result.Kind = "component"
+			} else if strings.Contains(href, "/apis/") {
+				result.Kind = "api"
+			} else {
+				result.Kind = "doc"
+			}
+		}
+
+		if result.Name != "" {
+			results = append(results, result)
+		}
+	})
+
+	return results, nil
+}
+
+func (p *Parser) extractHooks(doc *goquery.Document, docURL string) []*Hook {
+	var hooks []*Hook
+
+	doc.Find("h2, h3, .api-item, [id^='use']").Each(func(_ int, s *goquery.Selection) {
+		id, _ := s.Attr("id")
+		text := s.Text()
+
+		if !strings.HasPrefix(id, "use") && !strings.Contains(text, "use") {
+			return
+		}
+
+		hook := &Hook{}
+
+		nameEl := s.Find("code, .name").First()
+		if nameEl.Length() == 0 {
+			nameEl = s
+		}
+		hook.Name = strings.TrimSpace(nameEl.Text())
+		hook.Name = strings.TrimSuffix(hook.Name, "(")
+
+		if strings.HasPrefix(hook.Name, "use") {
+			hook.DocURL = docURL + "#" + hook.Name
+
+			next := s.Next()
+			for next.Length() > 0 && !next.Is("h2, h3") {
+				if next.Is("p") && hook.Doc == "" {
+					hook.Doc = strings.TrimSpace(next.Text())
+				} else if next.Is("pre, code") {
+					sig := strings.TrimSpace(next.Text())
+					if strings.HasPrefix(sig, hook.Name) {
+						hook.Signature = sig
+					}
+				}
+				next = next.Next()
+			}
+
+			hooks = append(hooks, hook)
+		}
+	})
+
+	return hooks
+}
+
+func (p *Parser) extractComponents(doc *goquery.Document, docURL string) []*Component {
+	var components []*Component
+
+	doc.Find("h2, h3, .api-item").Each(func(_ int, s *goquery.Selection) {
+		id, _ := s.Attr("id")
+		text := s.Text()
+
+		componentNames := []string{"Fragment", "Profiler", "StrictMode", "Suspense", "Transition", "Portal", "Component"}
+		isComponent := false
+		for _, name := range componentNames {
+			if id == name || strings.Contains(text, name) {
+				isComponent = true
+				break
+			}
+		}
+
+		if !isComponent {
+			return
+		}
+
+		comp := &Component{}
+
+		nameEl := s.Find("code, .name").First()
+		if nameEl.Length() == 0 {
+			nameEl = s
+		}
+		comp.Name = strings.TrimSpace(nameEl.Text())
+
+		comp.DocURL = docURL + "#" + comp.Name
+
+		next := s.Next()
+		for next.Length() > 0 && !next.Is("h2, h3") {
+			if next.Is("p") && comp.Doc == "" {
+				comp.Doc = strings.TrimSpace(next.Text())
+			}
+			next = next.Next()
+		}
+
+		if comp.Name != "" {
+			components = append(components, comp)
+		}
+	})
+
+	return components
+}
+
+func (p *Parser) extractAPIs(doc *goquery.Document, docURL string) []*API {
+	var apis []*API
+
+	doc.Find("h2, h3, .api-item").Each(func(_ int, s *goquery.Selection) {
+		id, _ := s.Attr("id")
+		text := s.Text()
+
+		apiNames := []string{"createContext", "createElement", "createFactory", "createRef", "forwardRef", "isValidElement", "lazy", "memo", "startTransition", "cloneElement", "Children"}
+		isAPI := false
+		for _, name := range apiNames {
+			if id == name || strings.Contains(text, name) {
+				isAPI = true
+				break
+			}
+		}
+
+		if !isAPI {
+			return
+		}
+
+		api := &API{}
+
+		nameEl := s.Find("code, .name").First()
+		if nameEl.Length() == 0 {
+			nameEl = s
+		}
+		api.Name = strings.TrimSpace(nameEl.Text())
+		api.Name = strings.TrimSuffix(api.Name, "(")
+
+		api.DocURL = docURL + "#" + api.Name
+
+		next := s.Next()
+		for next.Length() > 0 && !next.Is("h2, h3") {
+			if next.Is("p") && api.Doc == "" {
+				api.Doc = strings.TrimSpace(next.Text())
+			} else if next.Is("pre, code") {
+				sig := strings.TrimSpace(next.Text())
+				if strings.HasPrefix(sig, api.Name) || strings.Contains(sig, api.Name) {
+					api.Signature = sig
+				}
+			}
+			next = next.Next()
+		}
+
+		if api.Name != "" {
+			apis = append(apis, api)
+		}
+	})
+
+	return apis
+}
+
+func (p *Parser) extractDirectives(doc *goquery.Document, docURL string) []*Directive {
+	var directives []*Directive
+
+	directiveNames := []string{"use client", "use server"}
+
+	doc.Find("h2, h3, code, pre").Each(func(_ int, s *goquery.Selection) {
+		text := strings.TrimSpace(s.Text())
+
+		for _, name := range directiveNames {
+			if text == name || strings.Contains(text, "'"+name+"'") {
+				dir := &Directive{
+					Name:  name,
+					Usage: text,
+				}
+				dir.DocURL = docURL + "#" + strings.ReplaceAll(name, " ", "-")
+
+				next := s.Next()
+				for next.Length() > 0 && !next.Is("h2, h3") {
+					if next.Is("p") && dir.Doc == "" {
+						dir.Doc = strings.TrimSpace(next.Text())
+					}
+					next = next.Next()
+				}
+
+				directives = append(directives, dir)
+				break
+			}
+		}
+	})
+
+	return directives
+}
+
+func (p *Parser) ParseHookPage(html string, docURL string) (*Hook, error) {
+	doc, err := goquery.NewDocumentFromReader(strings.NewReader(html))
+	if err != nil {
+		return nil, err
+	}
+
+	hook := &Hook{
+		DocURL: docURL,
+	}
+
+	header := doc.Find("h1, .title").First()
+	hook.Name = strings.TrimSpace(header.Text())
+
+	hook.Doc = strings.TrimSpace(doc.Find(".content p, main p, article p").First().Text())
+
+	sigEl := doc.Find("pre code, .signature, code").First()
+	hook.Signature = strings.TrimSpace(sigEl.Text())
+
+	return hook, nil
+}
+
+func resolveURL(base string, href string) string {
+	if strings.HasPrefix(href, "http") {
+		return href
+	}
+
+	baseURL, err := url.Parse(base)
+	if err != nil {
+		return href
+	}
+
+	hrefURL, err := url.Parse(href)
+	if err != nil {
+		return href
+	}
+
+	return baseURL.ResolveReference(hrefURL).String()
+}

pkg/reactdocs/parser_test.go

@@ -0,0 +1,125 @@
+package reactdocs
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/PuerkitoBio/goquery"
+)
+
+const testReferencePageHTML = `
+<!DOCTYPE html>
+<html>
+<body>
+<h1>React Reference</h1>
+
+<h2 id="useState">useState</h2>
+<p>useState is a React Hook that lets you add a state variable to your component.</p>
+<pre><code>const [state, setState] = useState(initialState)</code></pre>
+
+<h2 id="useEffect">useEffect</h2>
+<p>useEffect is a React Hook that lets you synchronize a component with an external system.</p>
+<pre><code>useEffect(setup, dependencies?)</code></pre>
+
+<h2 id="Suspense">Suspense</h2>
+<p>Suspense lets you display a fallback until its children have finished loading.</p>
+
+<h2 id="createContext">createContext</h2>
+<p>createContext lets you create a Context that components can provide or read.</p>
+<pre><code>createContext(initialValue)</code></pre>
+</body>
+</html>
+`
+
+func TestParseReferencePage(t *testing.T) {
+	parser := NewParser()
+	ref, err := parser.ParseReferencePage(testReferencePageHTML, "https://react.dev/reference/react")
+	if err != nil {
+		t.Fatalf("ParseReferencePage failed: %v", err)
+	}
+
+	if len(ref.Hooks) == 0 {
+		t.Error("Expected at least one hook")
+	}
+
+	if len(ref.Components) == 0 {
+		t.Error("Expected at least one component")
+	}
+
+	if len(ref.APIs) == 0 {
+		t.Error("Expected at least one API")
+	}
+}
+
+func TestExtractHooks(t *testing.T) {
+	parser := NewParser()
+
+	doc, err := goquery.NewDocumentFromReader(strings.NewReader(testReferencePageHTML))
+	if err != nil {
+		t.Fatalf("Failed to parse HTML: %v", err)
+	}
+
+	hooks := parser.extractHooks(doc, "https://react.dev/reference/react")
+
+	if len(hooks) == 0 {
+		t.Fatal("Expected at least one hook")
+	}
+
+	found := false
+	for _, h := range hooks {
+		if h.Name == "useState" {
+			found = true
+			if h.Doc == "" {
+				t.Error("Expected useState to have documentation")
+			}
+			break
+		}
+	}
+
+	if !found {
+		t.Error("Expected to find useState hook")
+	}
+}
+
+func TestExtractComponents(t *testing.T) {
+	parser := NewParser()
+
+	doc, err := goquery.NewDocumentFromReader(strings.NewReader(testReferencePageHTML))
+	if err != nil {
+		t.Fatalf("Failed to parse HTML: %v", err)
+	}
+
+	components := parser.extractComponents(doc, "https://react.dev/reference/react")
+
+	found := false
+	for _, c := range components {
+		if c.Name == "Suspense" {
+			found = true
+			break
+		}
+	}
+
+	if !found {
+		t.Error("Expected to find Suspense component")
+	}
+}
+
+func TestResolveURL(t *testing.T) {
+	tests := []struct {
+		base     string
+		href     string
+		expected string
+	}{
+		{"https://react.dev", "/reference/react", "https://react.dev/reference/react"},
+		{"https://react.dev", "https://example.com/page", "https://example.com/page"},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.href, func(t *testing.T) {
+			got := resolveURL(tt.base, tt.href)
+			if got != tt.expected {
+				t.Errorf("resolveURL(%q, %q) = %q, want %q", tt.base, tt.href, got, tt.expected)
+			}
+		})
+	}
+}

pkg/reactdocs/types.go

@@ -0,0 +1,104 @@
+// Package reactdocs provides parsing and extraction for React documentation
+// from react.dev.
+package reactdocs
+
+import "time"
+
+// Reference represents the React API reference.
+type Reference struct {
+	Version    string       `json:"version"`
+	DocURL     string       `json:"doc_url"`
+	Hooks      []*Hook      `json:"hooks,omitempty"`
+	Components []*Component `json:"components,omitempty"`
+	APIs       []*API       `json:"apis,omitempty"`
+	Directives []*Directive `json:"directives,omitempty"`
+	FetchedAt  time.Time    `json:"fetched_at"`
+}
+
+// Hook represents a React hook.
+type Hook struct {
+	Name       string       `json:"name"`
+	Doc        string       `json:"doc,omitempty"`
+	Signature  string       `json:"signature,omitempty"`
+	Parameters []*Parameter `json:"parameters,omitempty"`
+	Returns    []*Return    `json:"returns,omitempty"`
+	Examples   []*Example   `json:"examples,omitempty"`
+	Category   string       `json:"category,omitempty"` // state, effect, ref, etc.
+	DocURL     string       `json:"doc_url"`
+	Deprecated string       `json:"deprecated,omitempty"`
+}
+
+// Component represents a React built-in component.
+type Component struct {
+	Name       string     `json:"name"`
+	Doc        string     `json:"doc,omitempty"`
+	Props      []*Prop    `json:"props,omitempty"`
+	Examples   []*Example `json:"examples,omitempty"`
+	DocURL     string     `json:"doc_url"`
+	Deprecated string     `json:"deprecated,omitempty"`
+}
+
+// Prop represents a component prop.
+type Prop struct {
+	Name       string `json:"name"`
+	Type       string `json:"type"`
+	Doc        string `json:"doc,omitempty"`
+	Required   bool   `json:"required"`
+	Default    string `json:"default,omitempty"`
+	Deprecated string `json:"deprecated,omitempty"`
+}
+
+// API represents a React API function.
+type API struct {
+	Name       string       `json:"name"`
+	Doc        string       `json:"doc,omitempty"`
+	Signature  string       `json:"signature,omitempty"`
+	Parameters []*Parameter `json:"parameters,omitempty"`
+	Returns    []*Return    `json:"returns,omitempty"`
+	Examples   []*Example   `json:"examples,omitempty"`
+	DocURL     string       `json:"doc_url"`
+	Deprecated string       `json:"deprecated,omitempty"`
+}
+
+// Directive represents a React directive (like 'use client', 'use server').
+type Directive struct {
+	Name     string     `json:"name"`
+	Doc      string     `json:"doc,omitempty"`
+	Usage    string     `json:"usage,omitempty"`
+	Examples []*Example `json:"examples,omitempty"`
+	DocURL   string     `json:"doc_url"`
+}
+
+// Parameter represents a function parameter.
+type Parameter struct {
+	Name     string `json:"name"`
+	Type     string `json:"type,omitempty"`
+	Doc      string `json:"doc,omitempty"`
+	Optional bool   `json:"optional"`
+	Default  string `json:"default,omitempty"`
+}
+
+// Return represents a return value.
+type Return struct {
+	Name string `json:"name,omitempty"`
+	Type string `json:"type,omitempty"`
+	Doc  string `json:"doc,omitempty"`
+}
+
+// Example represents a code example.
+type Example struct {
+	Title       string `json:"title,omitempty"`
+	Code        string `json:"code"`
+	Description string `json:"description,omitempty"`
+	Language    string `json:"language,omitempty"`
+}
+
+// SearchResult represents a search result.
+type SearchResult struct {
+	Name       string `json:"name"`
+	Kind       string `json:"kind"` // hook, component, api, directive
+	Doc        string `json:"doc,omitempty"`
+	DocURL     string `json:"doc_url"`
+	Score      int    `json:"score"`
+	Deprecated bool   `json:"deprecated"`
+}

pkg/rustdocs/parser.go

@@ -0,0 +1,633 @@
+package rustdocs
+
+import (
+	"net/url"
+	"regexp"
+	"strings"
+	"time"
+
+	"github.com/PuerkitoBio/goquery"
+)
+
+type Parser struct {
+	baseURL string
+}
+
+func NewParser() *Parser {
+	return &Parser{
+		baseURL: "https://docs.rs",
+	}
+}
+
+func (p *Parser) ParseCratePage(html string, docURL string) (*Crate, error) {
+	doc, err := goquery.NewDocumentFromReader(strings.NewReader(html))
+	if err != nil {
+		return nil, err
+	}
+
+	crate := &Crate{
+		DocURL:    docURL,
+		FetchedAt: time.Now(),
+	}
+
+	crate.Name = p.extractCrateName(doc)
+	crate.Version = p.extractVersion(doc)
+	crate.Description = p.extractDescription(doc)
+	crate.Repository = p.extractRepository(doc)
+
+	crate.Modules = p.extractModules(doc)
+	crate.Structs = p.extractStructs(doc)
+	crate.Enums = p.extractEnums(doc)
+	crate.Traits = p.extractTraits(doc)
+	crate.Functions = p.extractFunctions(doc)
+	crate.Macros = p.extractMacros(doc)
+	crate.Constants = p.extractConstants(doc)
+	crate.Statics = p.extractStatics(doc)
+
+	return crate, nil
+}
+
+func (p *Parser) ParseItemPage(html string, docURL string) (*Symbol, error) {
+	doc, err := goquery.NewDocumentFromReader(strings.NewReader(html))
+	if err != nil {
+		return nil, err
+	}
+
+	symbol := &Symbol{
+		DocURL: docURL,
+	}
+
+	symbol.Name = p.extractItemName(doc)
+	symbol.Path = p.extractItemPath(doc, docURL)
+	symbol.Kind = p.extractItemKind(doc)
+	symbol.Signature = p.extractItemSignature(doc)
+	symbol.Doc = p.extractItemDoc(doc)
+
+	return symbol, nil
+}
+
+func (p *Parser) ParseSearchResults(html string) ([]*SearchResult, error) {
+	doc, err := goquery.NewDocumentFromReader(strings.NewReader(html))
+	if err != nil {
+		return nil, err
+	}
+
+	var results []*SearchResult
+
+	doc.Find("#results .search-results a").Each(func(i int, s *goquery.Selection) {
+		result := &SearchResult{}
+
+		classes, _ := s.Attr("class")
+		result.Kind = extractKindFromClasses(classes)
+
+		nameEl := s.Find(".result-name")
+		result.Name = strings.TrimSpace(nameEl.Find(".method, .struct, .fn, .trait, .enum, .mod, .macro, .const, .static, .attr").Text())
+		if result.Name == "" {
+			nameText := nameEl.Text()
+			result.Name = strings.TrimSpace(strings.Split(nameText, "\n")[0])
+		}
+
+		var pathParts []string
+		nameEl.Find(".path span").Each(func(_ int, span *goquery.Selection) {
+			part := strings.TrimSpace(span.Text())
+			if part != "" {
+				pathParts = append(pathParts, part)
+			}
+		})
+		result.Path = strings.Join(pathParts, "::")
+
+		result.Description = strings.TrimSpace(s.Find(".desc").Text())
+
+		if href, exists := s.Attr("href"); exists {
+			if strings.HasPrefix(href, "http") {
+				result.DocURL = href
+			} else {
+				u, err := url.Parse("https://docs.rs")
+				if err == nil {
+					u.Path = href
+					result.DocURL = u.String()
+				}
+			}
+		}
+
+		stabilityEl := s.Find(".stab")
+		if stabilityEl.Length() > 0 {
+			if stabilityEl.HasClass("unstable") || stabilityEl.HasClass("experimental") {
+				result.IsExperimental = true
+			}
+			result.Stability = strings.TrimSpace(stabilityEl.Text())
+		}
+
+		results = append(results, result)
+	})
+
+	return results, nil
+}
+
+func (p *Parser) extractCrateName(doc *goquery.Document) string {
+	title := doc.Find(".main-heading h1").Text()
+	title = strings.TrimSpace(title)
+
+	if strings.HasPrefix(title, "Crate ") {
+		return strings.TrimPrefix(title, "Crate ")
+	}
+	if strings.HasPrefix(title, "Module ") {
+		return strings.TrimPrefix(title, "Module ")
+	}
+
+	h1 := doc.Find("h1").First().Text()
+	h1 = strings.TrimSpace(h1)
+	if strings.HasPrefix(h1, "Crate ") {
+		return strings.TrimPrefix(h1, "Crate ")
+	}
+
+	return title
+}
+
+func (p *Parser) extractVersion(doc *goquery.Document) string {
+	since := doc.Find(".since").Text()
+	if since != "" {
+		re := regexp.MustCompile(`\d+\.\d+\.\d+`)
+		if match := re.FindString(since); match != "" {
+			return match
+		}
+	}
+
+	subHeading := doc.Find(".sub-heading").Text()
+	re := regexp.MustCompile(`v?(\d+\.\d+\.\d+)`)
+	if match := re.FindStringSubmatch(subHeading); len(match) > 1 {
+		return match[1]
+	}
+
+	return ""
+}
+
+func (p *Parser) extractDescription(doc *goquery.Document) string {
+	topDoc := doc.Find(".top-doc .docblock").First()
+	if topDoc.Length() > 0 {
+		return strings.TrimSpace(topDoc.Text())
+	}
+
+	topDoc = doc.Find(".docblock").First()
+	if topDoc.Length() > 0 {
+		return strings.TrimSpace(topDoc.Text())
+	}
+
+	return ""
+}
+
+func (p *Parser) extractRepository(doc *goquery.Document) string {
+	srcLink := doc.Find("a.src")
+	if srcLink.Length() > 0 {
+		if href, exists := srcLink.Attr("href"); exists {
+			if strings.Contains(href, "github.com") {
+				re := regexp.MustCompile(`https://github\.com/[^/]+/[^/]+`)
+				if match := re.FindString(href); match != "" {
+					return match
+				}
+			}
+		}
+	}
+	return ""
+}
+
+func (p *Parser) extractItemName(doc *goquery.Document) string {
+	h1 := doc.Find(".main-heading h1").Text()
+	h1 = strings.TrimSpace(h1)
+
+	for _, prefix := range []string{"Struct ", "Enum ", "Trait ", "Fn ", "Macro ", "Const ", "Static ", "Module ", "Type "} {
+		if strings.HasPrefix(h1, prefix) {
+			return strings.TrimPrefix(h1, prefix)
+		}
+	}
+
+	return h1
+}
+
+func (p *Parser) extractItemPath(doc *goquery.Document, docURL string) string {
+	breadcrumbs := doc.Find(".rustdoc-breadcrumbs").Text()
+	breadcrumbs = strings.TrimSpace(breadcrumbs)
+	breadcrumbs = strings.ReplaceAll(breadcrumbs, "\n", "")
+	breadcrumbs = strings.ReplaceAll(breadcrumbs, "  ", " ")
+	breadcrumbs = strings.TrimSpace(breadcrumbs)
+
+	if breadcrumbs != "" {
+		return breadcrumbs
+	}
+
+	if docURL != "" {
+		u, err := url.Parse(docURL)
+		if err == nil {
+			path := strings.TrimPrefix(u.Path, "/")
+			path = strings.TrimSuffix(path, "/index.html")
+			path = strings.TrimSuffix(path, ".html")
+			path = strings.ReplaceAll(path, "/", "::")
+			return path
+		}
+	}
+
+	return ""
+}
+
+func (p *Parser) extractItemKind(doc *goquery.Document) ItemKind {
+	h1 := doc.Find(".main-heading h1 span").First()
+	if h1.Length() > 0 {
+		class, _ := h1.Attr("class")
+		switch {
+		case strings.Contains(class, "struct"):
+			return ItemKindStruct
+		case strings.Contains(class, "enum"):
+			return ItemKindEnum
+		case strings.Contains(class, "trait"):
+			return ItemKindTrait
+		case strings.Contains(class, "fn"):
+			return ItemKindFn
+		case strings.Contains(class, "macro"):
+			return ItemKindMacro
+		case strings.Contains(class, "const"):
+			return ItemKindConst
+		case strings.Contains(class, "static"):
+			return ItemKindStatic
+		case strings.Contains(class, "mod"):
+			return ItemKindMod
+		case strings.Contains(class, "type"):
+			return ItemKindType
+		}
+	}
+
+	title := doc.Find(".main-heading h1").Text()
+	switch {
+	case strings.HasPrefix(title, "Struct "):
+		return ItemKindStruct
+	case strings.HasPrefix(title, "Enum "):
+		return ItemKindEnum
+	case strings.HasPrefix(title, "Trait "):
+		return ItemKindTrait
+	case strings.HasPrefix(title, "Fn ") || strings.HasPrefix(title, "Function "):
+		return ItemKindFn
+	case strings.HasPrefix(title, "Macro "):
+		return ItemKindMacro
+	case strings.HasPrefix(title, "Const "):
+		return ItemKindConst
+	case strings.HasPrefix(title, "Static "):
+		return ItemKindStatic
+	case strings.HasPrefix(title, "Module "):
+		return ItemKindMod
+	}
+
+	return ""
+}
+
+func (p *Parser) extractItemSignature(doc *goquery.Document) string {
+	sig := doc.Find("pre.rust.item-decl").Text()
+	sig = strings.TrimSpace(sig)
+	if sig != "" {
+		return sig
+	}
+
+	sig = doc.Find("pre.rust").First().Text()
+	return strings.TrimSpace(sig)
+}
+
+func (p *Parser) extractItemDoc(doc *goquery.Document) string {
+	docblock := doc.Find(".top-doc .docblock").First()
+	if docblock.Length() > 0 {
+		return strings.TrimSpace(docblock.Text())
+	}
+
+	docblock = doc.Find(".docblock").First()
+	if docblock.Length() > 0 {
+		return strings.TrimSpace(docblock.Text())
+	}
+
+	return ""
+}
+
+func (p *Parser) extractModules(doc *goquery.Document) []*Module {
+	var modules []*Module
+
+	doc.Find(".item-table .mod, .module-item .mod").Each(func(_ int, s *goquery.Selection) {
+		mod := &Module{}
+
+		mod.Name = strings.TrimSpace(s.Find("a.mod").Text())
+		if mod.Name == "" {
+			mod.Name = strings.TrimSpace(s.Find("a").First().Text())
+		}
+
+		if href, exists := s.Find("a").First().Attr("href"); exists {
+			mod.DocURL = resolveURL(p.baseURL, href)
+		}
+
+		mod.Doc = strings.TrimSpace(s.Find(".desc, .item-desc").Text())
+
+		mod.IsExperimental = s.Find(".stab.unstable, .stab.experimental").Length() > 0
+
+		if mod.Name != "" {
+			modules = append(modules, mod)
+		}
+	})
+
+	return modules
+}
+
+func (p *Parser) extractStructs(doc *goquery.Document) []*Struct {
+	var structs []*Struct
+
+	doc.Find(".item-table .struct, .struct").Each(func(_ int, s *goquery.Selection) {
+		st := &Struct{}
+
+		st.Name = strings.TrimSpace(s.Find("a.struct").Text())
+		if st.Name == "" {
+			st.Name = strings.TrimSpace(s.Find("a").First().Text())
+		}
+
+		if href, exists := s.Find("a").First().Attr("href"); exists {
+			st.DocURL = resolveURL(p.baseURL, href)
+		}
+
+		st.Doc = strings.TrimSpace(s.Find(".desc, .item-desc").Text())
+		st.IsExperimental = s.Find(".stab.unstable, .stab.experimental").Length() > 0
+
+		if st.Name != "" {
+			structs = append(structs, st)
+		}
+	})
+
+	return structs
+}
+
+func (p *Parser) extractEnums(doc *goquery.Document) []*Enum {
+	var enums []*Enum
+
+	doc.Find(".item-table .enum, .enum").Each(func(_ int, s *goquery.Selection) {
+		e := &Enum{}
+
+		e.Name = strings.TrimSpace(s.Find("a.enum").Text())
+		if e.Name == "" {
+			e.Name = strings.TrimSpace(s.Find("a").First().Text())
+		}
+
+		if href, exists := s.Find("a").First().Attr("href"); exists {
+			e.DocURL = resolveURL(p.baseURL, href)
+		}
+
+		e.Doc = strings.TrimSpace(s.Find(".desc, .item-desc").Text())
+		e.IsExperimental = s.Find(".stab.unstable, .stab.experimental").Length() > 0
+
+		if e.Name != "" {
+			enums = append(enums, e)
+		}
+	})
+
+	return enums
+}
+
+func (p *Parser) extractTraits(doc *goquery.Document) []*Trait {
+	var traits []*Trait
+
+	doc.Find(".item-table .trait, .trait").Each(func(_ int, s *goquery.Selection) {
+		t := &Trait{}
+
+		t.Name = strings.TrimSpace(s.Find("a.trait").Text())
+		if t.Name == "" {
+			t.Name = strings.TrimSpace(s.Find("a").First().Text())
+		}
+
+		if href, exists := s.Find("a").First().Attr("href"); exists {
+			t.DocURL = resolveURL(p.baseURL, href)
+		}
+
+		t.Doc = strings.TrimSpace(s.Find(".desc, .item-desc").Text())
+		t.IsExperimental = s.Find(".stab.unstable, .stab.experimental").Length() > 0
+
+		if t.Name != "" {
+			traits = append(traits, t)
+		}
+	})
+
+	return traits
+}
+
+func (p *Parser) extractFunctions(doc *goquery.Document) []*Func {
+	var funcs []*Func
+
+	doc.Find(".item-table .fn, .fn, .function").Each(func(_ int, s *goquery.Selection) {
+		f := &Func{}
+
+		f.Name = strings.TrimSpace(s.Find("a.fn").Text())
+		if f.Name == "" {
+			f.Name = strings.TrimSpace(s.Find("a").First().Text())
+		}
+
+		if href, exists := s.Find("a").First().Attr("href"); exists {
+			f.DocURL = resolveURL(p.baseURL, href)
+		}
+
+		f.Doc = strings.TrimSpace(s.Find(".desc, .item-desc").Text())
+		f.IsExperimental = s.Find(".stab.unstable, .stab.experimental").Length() > 0
+		f.IsUnsafe = strings.Contains(s.Text(), "unsafe")
+
+		if f.Name != "" {
+			funcs = append(funcs, f)
+		}
+	})
+
+	return funcs
+}
+
+func (p *Parser) extractMacros(doc *goquery.Document) []*Macro {
+	var macros []*Macro
+
+	doc.Find(".item-table .macro, .macro").Each(func(_ int, s *goquery.Selection) {
+		m := &Macro{}
+
+		m.Name = strings.TrimSpace(s.Find("a.macro").Text())
+		if m.Name == "" {
+			m.Name = strings.TrimSpace(s.Find("a").First().Text())
+		}
+
+		if href, exists := s.Find("a").First().Attr("href"); exists {
+			m.DocURL = resolveURL(p.baseURL, href)
+		}
+
+		m.Doc = strings.TrimSpace(s.Find(".desc, .item-desc").Text())
+		m.IsExperimental = s.Find(".stab.unstable, .stab.experimental").Length() > 0
+
+		if m.Name != "" {
+			macros = append(macros, m)
+		}
+	})
+
+	return macros
+}
+
+func (p *Parser) extractConstants(doc *goquery.Document) []*Const {
+	var constants []*Const
+
+	doc.Find(".item-table .constant, .constant").Each(func(_ int, s *goquery.Selection) {
+		c := &Const{}
+
+		c.Name = strings.TrimSpace(s.Find("a.constant").Text())
+		if c.Name == "" {
+			c.Name = strings.TrimSpace(s.Find("a").First().Text())
+		}
+
+		if href, exists := s.Find("a").First().Attr("href"); exists {
+			c.DocURL = resolveURL(p.baseURL, href)
+		}
+
+		c.Doc = strings.TrimSpace(s.Find(".desc, .item-desc").Text())
+		c.IsExperimental = s.Find(".stab.unstable, .stab.experimental").Length() > 0
+
+		if c.Name != "" {
+			constants = append(constants, c)
+		}
+	})
+
+	return constants
+}
+
+func (p *Parser) extractStatics(doc *goquery.Document) []*Static {
+	var statics []*Static
+
+	doc.Find(".item-table .static, .static").Each(func(_ int, s *goquery.Selection) {
+		st := &Static{}
+
+		st.Name = strings.TrimSpace(s.Find("a.static").Text())
+		if st.Name == "" {
+			st.Name = strings.TrimSpace(s.Find("a").First().Text())
+		}
+
+		if href, exists := s.Find("a").First().Attr("href"); exists {
+			st.DocURL = resolveURL(p.baseURL, href)
+		}
+
+		st.Doc = strings.TrimSpace(s.Find(".desc, .item-desc").Text())
+		st.IsExperimental = s.Find(".stab.unstable, .stab.experimental").Length() > 0
+
+		if st.Name != "" {
+			statics = append(statics, st)
+		}
+	})
+
+	return statics
+}
+
+func (p *Parser) ExtractMethods(doc *goquery.Document) []*Method {
+	var methods []*Method
+
+	doc.Find(".impl-items .method-toggle, details.method-toggle").Each(func(_ int, s *goquery.Selection) {
+		m := &Method{}
+
+		m.Name = strings.TrimSpace(s.Find(".fn, .method, h4.code-header").Text())
+		if m.Name == "" {
+			section := s.Find("section.method")
+			m.Name = strings.TrimSpace(section.Find(".fn").Text())
+		}
+
+		sig := s.Find("pre, .code-header, h4.code-header")
+		m.Signature = strings.TrimSpace(sig.Text())
+
+		m.Doc = strings.TrimSpace(s.Find(".docblock").Text())
+
+		m.IsUnsafe = strings.Contains(m.Signature, "unsafe")
+		m.IsAsync = strings.Contains(m.Signature, "async")
+		m.IsConst = strings.Contains(m.Signature, "const")
+
+		m.IsExperimental = s.Find(".stab.unstable, .stab.experimental").Length() > 0
+
+		if m.Name != "" {
+			methods = append(methods, m)
+		}
+	})
+
+	return methods
+}
+
+func (p *Parser) ExtractStructFields(doc *goquery.Document) []*Field {
+	var fields []*Field
+
+	doc.Find(".struct .fields tr, .struct-member").Each(func(_ int, s *goquery.Selection) {
+		f := &Field{}
+
+		f.Name = strings.TrimSpace(s.Find(".structfield, td:first-child").Text())
+		f.Type = strings.TrimSpace(s.Find(".type, td:nth-child(2)").Text())
+		f.Doc = strings.TrimSpace(s.Find(".docblock, td:last-child").Text())
+		f.IsPub = strings.Contains(s.Text(), "pub")
+
+		if f.Name != "" {
+			fields = append(fields, f)
+		}
+	})
+
+	return fields
+}
+
+func (p *Parser) ExtractEnumVariants(doc *goquery.Document) []*Variant {
+	var variants []*Variant
+
+	doc.Find(".enum .variants li, .variant").Each(func(_ int, s *goquery.Selection) {
+		v := &Variant{}
+
+		v.Name = strings.TrimSpace(s.Find("a, .variant-name").Text())
+		if v.Name == "" {
+			v.Name = strings.TrimSpace(s.Text())
+		}
+
+		v.Doc = strings.TrimSpace(s.Find(".docblock").Text())
+
+		sig := s.Text()
+		v.IsTuple = strings.Contains(sig, "(") && !strings.Contains(sig, "{")
+		v.IsStruct = strings.Contains(sig, "{")
+		v.IsUnit = !v.IsTuple && !v.IsStruct
+
+		if v.Name != "" {
+			variants = append(variants, v)
+		}
+	})
+
+	return variants
+}
+
+func extractKindFromClasses(classes string) string {
+	classList := strings.Fields(classes)
+	for _, c := range classList {
+		switch {
+		case strings.HasPrefix(c, "result-"):
+			kind := strings.TrimPrefix(c, "result-")
+			switch kind {
+			case "struct", "enum", "trait", "fn", "macro", "const", "static", "mod", "type", "primitive", "keyword", "attr":
+				return kind
+			case "method":
+				return "fn"
+			case "externcrate":
+				return "mod"
+			}
+		}
+	}
+	return ""
+}
+
+func resolveURL(base string, href string) string {
+	if strings.HasPrefix(href, "http") {
+		return href
+	}
+
+	baseURL, err := url.Parse(base)
+	if err != nil {
+		return href
+	}
+
+	hrefURL, err := url.Parse(href)
+	if err != nil {
+		return href
+	}
+
+	return baseURL.ResolveReference(hrefURL).String()
+}
+
+func cleanText(text string) string {
+	re := regexp.MustCompile(`\s+`)
+	text = re.ReplaceAllString(text, " ")
+	return strings.TrimSpace(text)
+}

pkg/rustdocs/parser_test.go

@@ -0,0 +1,265 @@
+package rustdocs
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/PuerkitoBio/goquery"
+)
+
+const testStructPageHTML = `
+<!DOCTYPE html>
+<html>
+<body>
+<main>
+<div class="main-heading">
+  <div class="rustdoc-breadcrumbs"><a href="../index.html">std</a>::<wbr><a href="index.html">simd</a></div>
+  <h1>Struct <span class="struct">Mask</span></h1>
+  <span class="sub-heading"><span class="since" title="Stable since Rust version 1.0.0">1.0.0</span></span>
+</div>
+<pre class="rust item-decl"><code>pub struct Mask&lt;T, const N: usize&gt;(<span class="comment">/* private fields */</span>)</code></pre>
+<details class="toggle top-doc" open="">
+  <div class="docblock"><p>A SIMD vector mask for N elements.</p></div>
+</details>
+<h2 id="implementations">Implementations</h2>
+<details class="toggle implementors-toggle" open="">
+  <details class="toggle method-toggle" open="">
+    <section id="method.test" class="method">
+      <h4 class="code-header">pub fn <a href="#method.test" class="fn">test</a>(&amp;self, index: usize) -&gt; bool</h4>
+    </section>
+    <div class="docblock"><p>Tests the value of the specified element.</p></div>
+  </details>
+</details>
+</main>
+</body>
+</html>
+`
+
+const testSearchHTML = `
+<div id="results">
+<ul class="search-results active">
+<a class="result-method" href="../std/simd/struct.Mask.html#method.test">
+  <span class="result-name">
+    <span class="typename">method</span>
+    <div class="path"><span>std::</span><span>simd::</span><span class="method">Mask::</span><span class="fn">test</span></div>
+  </span>
+  <div class="desc">Tests the value of the specified element.</div>
+</a>
+<a class="result-struct" href="../std/vec/struct.Vec.html">
+  <span class="result-name">
+    <span class="typename">struct</span>
+    <div class="path"><span>std::</span><span>vec::</span><span class="struct">Vec</span></div>
+  </span>
+  <div class="desc">A contiguous growable array type.</div>
+</a>
+<a class="result-fn" href="../std/io/fn.stdout.html">
+  <span class="result-name">
+    <span class="typename">fn</span>
+    <div class="path"><span>std::</span><span>io::</span><span class="fn">stdout</span></div>
+  </span>
+  <div class="desc">Constructs a new handle to the standard output.</div>
+</a>
+</ul>
+</div>
+`
+
+const testCratePageHTML = `
+<!DOCTYPE html>
+<html>
+<body>
+<main>
+<div class="main-heading">
+  <h1>Crate <span>serde</span></h1>
+  <span class="sub-heading"><span class="since">1.0.0</span></span>
+</div>
+<details class="toggle top-doc">
+  <div class="docblock"><p>A framework for serializing and deserializing Rust data structures.</p></div>
+</details>
+<h2 id="modules">Modules</h2>
+<div class="item-table">
+  <div class="module-item"><a class="mod" href="de/index.html">de</a><div class="desc">Deserialize implementation.</div></div>
+</div>
+<h2 id="structs">Structs</h2>
+<div class="item-table">
+  <div class="struct"><a class="struct" href="struct.Serializer.html">Serializer</a><div class="desc">A structure for serializing Rust values.</div></div>
+</div>
+<h2 id="enums">Enums</h2>
+<div class="item-table">
+  <div class="enum"><a class="enum" href="enum.Error.html">Error</a><div class="desc">Errors during serialization.</div></div>
+</div>
+</main>
+</body>
+</html>
+`
+
+func TestParseItemPage(t *testing.T) {
+	parser := NewParser()
+	symbol, err := parser.ParseItemPage(testStructPageHTML, "https://docs.rs/std/simd/struct.Mask.html")
+	if err != nil {
+		t.Fatalf("ParseItemPage failed: %v", err)
+	}
+
+	if symbol.Name != "Mask" {
+		t.Errorf("Expected name 'Mask', got '%s'", symbol.Name)
+	}
+
+	if symbol.Kind != ItemKindStruct {
+		t.Errorf("Expected kind 'struct', got '%s'", symbol.Kind)
+	}
+
+	if symbol.Doc == "" {
+		t.Error("Expected non-empty doc")
+	}
+
+	if !strings.Contains(symbol.Signature, "struct Mask") {
+		t.Errorf("Expected signature to contain 'struct Mask', got '%s'", symbol.Signature)
+	}
+}
+
+func TestParseSearchResults(t *testing.T) {
+	parser := NewParser()
+	results, err := parser.ParseSearchResults(testSearchHTML)
+	if err != nil {
+		t.Fatalf("ParseSearchResults failed: %v", err)
+	}
+
+	if len(results) < 3 {
+		t.Fatalf("Expected at least 3 results, got %d", len(results))
+	}
+
+	method := results[0]
+	if method.Kind != "fn" {
+		t.Errorf("Expected kind 'fn' for method, got '%s'", method.Kind)
+	}
+	if method.Description == "" {
+		t.Error("Expected non-empty description")
+	}
+
+	structResult := results[1]
+	if structResult.Kind != "struct" {
+		t.Errorf("Expected kind 'struct', got '%s'", structResult.Kind)
+	}
+
+	fnResult := results[2]
+	if fnResult.Kind != "fn" {
+		t.Errorf("Expected kind 'fn', got '%s'", fnResult.Kind)
+	}
+}
+
+func TestParseCratePage(t *testing.T) {
+	parser := NewParser()
+	crate, err := parser.ParseCratePage(testCratePageHTML, "https://docs.rs/serde")
+	if err != nil {
+		t.Fatalf("ParseCratePage failed: %v", err)
+	}
+
+	if crate.Name == "" {
+		t.Error("Expected non-empty name")
+	}
+
+	if crate.Description == "" {
+		t.Error("Expected non-empty description")
+	}
+}
+
+func TestExtractKindFromClasses(t *testing.T) {
+	tests := []struct {
+		classes  string
+		expected string
+	}{
+		{"result-struct", "struct"},
+		{"result-enum", "enum"},
+		{"result-trait", "trait"},
+		{"result-fn", "fn"},
+		{"result-macro", "macro"},
+		{"result-const", "const"},
+		{"result-static", "static"},
+		{"result-mod", "mod"},
+		{"result-method", "fn"},
+		{"result-externcrate", "mod"},
+		{"unknown-class", ""},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.classes, func(t *testing.T) {
+			got := extractKindFromClasses(tt.classes)
+			if got != tt.expected {
+				t.Errorf("extractKindFromClasses(%q) = %q, want %q", tt.classes, got, tt.expected)
+			}
+		})
+	}
+}
+
+func TestResolveURL(t *testing.T) {
+	tests := []struct {
+		base     string
+		href     string
+		expected string
+	}{
+		{"https://docs.rs", "/serde/struct.Serializer.html", "https://docs.rs/serde/struct.Serializer.html"},
+		{"https://docs.rs", "https://example.com/page", "https://example.com/page"},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.href, func(t *testing.T) {
+			got := resolveURL(tt.base, tt.href)
+			if got != tt.expected {
+				t.Errorf("resolveURL(%q, %q) = %q, want %q", tt.base, tt.href, got, tt.expected)
+			}
+		})
+	}
+}
+
+func TestCleanText(t *testing.T) {
+	tests := []struct {
+		input    string
+		expected string
+	}{
+		{"  hello   world  ", "hello world"},
+		{"single", "single"},
+		{"\n\ttabs\t\n", "tabs"},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.input, func(t *testing.T) {
+			if got := cleanText(tt.input); got != tt.expected {
+				t.Errorf("cleanText(%q) = %q, want %q", tt.input, got, tt.expected)
+			}
+		})
+	}
+}
+
+func TestExtractItemPath(t *testing.T) {
+	parser := NewParser()
+
+	html := `<div class="rustdoc-breadcrumbs"><a href="../index.html">std</a>::<a href="index.html">simd</a></div>`
+	doc, err := goquery.NewDocumentFromReader(strings.NewReader(html))
+	if err != nil {
+		t.Fatalf("Failed to parse HTML: %v", err)
+	}
+
+	got := parser.extractItemPath(doc, "https://docs.rs/std/simd/struct.Mask.html")
+
+	if !strings.Contains(got, "std") || !strings.Contains(got, "simd") {
+		t.Errorf("extractItemPath() = %q, expected to contain std and simd", got)
+	}
+}
+
+func TestExtractMethods(t *testing.T) {
+	parser := NewParser()
+
+	doc, err := goquery.NewDocumentFromReader(strings.NewReader(testStructPageHTML))
+	if err != nil {
+		t.Fatalf("Failed to parse HTML: %v", err)
+	}
+
+	methods := parser.ExtractMethods(doc)
+	if len(methods) == 0 {
+		t.Error("Expected at least one method")
+		return
+	}
+
+	if methods[0].Name == "" {
+		t.Error("Expected non-empty method name")
+	}
+}

pkg/rustdocs/types.go

@@ -0,0 +1,182 @@
+package rustdocs
+
+import "time"
+
+type Crate struct {
+	Name        string    `json:"name"`
+	Version     string    `json:"version"`
+	Description string    `json:"description"`
+	Modules     []*Module `json:"modules,omitempty"`
+	Structs     []*Struct `json:"structs,omitempty"`
+	Enums       []*Enum   `json:"enums,omitempty"`
+	Traits      []*Trait  `json:"traits,omitempty"`
+	Functions   []*Func   `json:"functions,omitempty"`
+	Macros      []*Macro  `json:"macros,omitempty"`
+	Constants   []*Const  `json:"constants,omitempty"`
+	Statics     []*Static `json:"statics,omitempty"`
+	DocURL      string    `json:"doc_url"`
+	Repository  string    `json:"repository,omitempty"`
+	License     string    `json:"license,omitempty"`
+	FetchedAt   time.Time `json:"fetched_at"`
+}
+
+type Module struct {
+	Name           string `json:"name"`
+	Path           string `json:"path"`
+	Doc            string `json:"doc,omitempty"`
+	DocURL         string `json:"doc_url"`
+	Stability      string `json:"stability,omitempty"`
+	IsExperimental bool   `json:"is_experimental"`
+}
+
+type Struct struct {
+	Name           string    `json:"name"`
+	Path           string    `json:"path"`
+	Doc            string    `json:"doc,omitempty"`
+	Declaration    string    `json:"declaration"`
+	Fields         []*Field  `json:"fields,omitempty"`
+	Methods        []*Method `json:"methods,omitempty"`
+	TraitImpls     []string  `json:"trait_impls,omitempty"`
+	DocURL         string    `json:"doc_url"`
+	Stability      string    `json:"stability,omitempty"`
+	IsExperimental bool      `json:"is_experimental"`
+}
+
+type Enum struct {
+	Name           string     `json:"name"`
+	Path           string     `json:"path"`
+	Doc            string     `json:"doc,omitempty"`
+	Declaration    string     `json:"declaration"`
+	Variants       []*Variant `json:"variants,omitempty"`
+	Methods        []*Method  `json:"methods,omitempty"`
+	DocURL         string     `json:"doc_url"`
+	Stability      string     `json:"stability,omitempty"`
+	IsExperimental bool       `json:"is_experimental"`
+}
+
+type Trait struct {
+	Name             string    `json:"name"`
+	Path             string    `json:"path"`
+	Doc              string    `json:"doc,omitempty"`
+	Declaration      string    `json:"declaration"`
+	Methods          []*Method `json:"methods,omitempty"`
+	AssociatedTypes  []string  `json:"associated_types,omitempty"`
+	AssociatedConsts []string  `json:"associated_consts,omitempty"`
+	DocURL           string    `json:"doc_url"`
+	Stability        string    `json:"stability,omitempty"`
+	IsExperimental   bool      `json:"is_experimental"`
+}
+
+type Func struct {
+	Name           string `json:"name"`
+	Path           string `json:"path"`
+	Doc            string `json:"doc,omitempty"`
+	Signature      string `json:"signature"`
+	DocURL         string `json:"doc_url"`
+	Stability      string `json:"stability,omitempty"`
+	IsExperimental bool   `json:"is_experimental"`
+	IsUnsafe       bool   `json:"is_unsafe"`
+	IsConst        bool   `json:"is_const"`
+	IsAsync        bool   `json:"is_async"`
+}
+
+type Macro struct {
+	Name           string   `json:"name"`
+	Path           string   `json:"path"`
+	Doc            string   `json:"doc,omitempty"`
+	Signature      string   `json:"signature,omitempty"`
+	DocURL         string   `json:"doc_url"`
+	Stability      string   `json:"stability,omitempty"`
+	IsExperimental bool     `json:"is_experimental"`
+	Examples       []string `json:"examples,omitempty"`
+}
+
+type Const struct {
+	Name           string `json:"name"`
+	Path           string `json:"path"`
+	Doc            string `json:"doc,omitempty"`
+	Type           string `json:"type,omitempty"`
+	Value          string `json:"value,omitempty"`
+	DocURL         string `json:"doc_url"`
+	Stability      string `json:"stability,omitempty"`
+	IsExperimental bool   `json:"is_experimental"`
+}
+
+type Static struct {
+	Name           string `json:"name"`
+	Path           string `json:"path"`
+	Doc            string `json:"doc,omitempty"`
+	Type           string `json:"type,omitempty"`
+	IsMutable      bool   `json:"is_mutable"`
+	DocURL         string `json:"doc_url"`
+	Stability      string `json:"stability,omitempty"`
+	IsExperimental bool   `json:"is_experimental"`
+}
+
+type Field struct {
+	Name       string `json:"name"`
+	Type       string `json:"type"`
+	Doc        string `json:"doc,omitempty"`
+	IsPub      bool   `json:"is_pub"`
+	Visibility string `json:"visibility,omitempty"`
+}
+
+type Variant struct {
+	Name     string   `json:"name"`
+	Doc      string   `json:"doc,omitempty"`
+	Fields   []string `json:"fields,omitempty"`
+	IsTuple  bool     `json:"is_tuple"`
+	IsStruct bool     `json:"is_struct"`
+	IsUnit   bool     `json:"is_unit"`
+}
+
+type Method struct {
+	Name           string   `json:"name"`
+	Doc            string   `json:"doc,omitempty"`
+	Signature      string   `json:"signature"`
+	Receiver       string   `json:"receiver,omitempty"`
+	IsUnsafe       bool     `json:"is_unsafe"`
+	IsConst        bool     `json:"is_const"`
+	IsAsync        bool     `json:"is_async"`
+	IsDefault      bool     `json:"is_default"`
+	Examples       []string `json:"examples,omitempty"`
+	Stability      string   `json:"stability,omitempty"`
+	IsExperimental bool     `json:"is_experimental"`
+}
+
+type SearchResult struct {
+	Name           string `json:"name"`
+	Path           string `json:"path"`
+	Kind           string `json:"kind"`
+	Description    string `json:"description,omitempty"`
+	DocURL         string `json:"doc_url"`
+	Stability      string `json:"stability,omitempty"`
+	IsExperimental bool   `json:"is_experimental"`
+}
+
+type ItemKind string
+
+const (
+	ItemKindStruct    ItemKind = "struct"
+	ItemKindEnum      ItemKind = "enum"
+	ItemKindTrait     ItemKind = "trait"
+	ItemKindFn        ItemKind = "fn"
+	ItemKindMacro     ItemKind = "macro"
+	ItemKindConst     ItemKind = "const"
+	ItemKindStatic    ItemKind = "static"
+	ItemKindMod       ItemKind = "mod"
+	ItemKindType      ItemKind = "type"
+	ItemKindUnion     ItemKind = "union"
+	ItemKindPrimitive ItemKind = "primitive"
+	ItemKindKeyword   ItemKind = "keyword"
+	ItemKindAttr      ItemKind = "attr"
+)
+
+type Symbol struct {
+	Name      string   `json:"name"`
+	Kind      ItemKind `json:"kind"`
+	Path      string   `json:"path"`
+	Signature string   `json:"signature,omitempty"`
+	Doc       string   `json:"doc,omitempty"`
+	DocURL    string   `json:"doc_url"`
+}

pkg/springdocs/parser.go

@@ -0,0 +1,310 @@
+package springdocs
+
+import (
+	"net/url"
+	"strings"
+	"time"
+
+	"github.com/PuerkitoBio/goquery"
+)
+
+type Parser struct {
+	baseURL string
+}
+
+func NewParser() *Parser {
+	return &Parser{
+		baseURL: "https://docs.spring.io",
+	}
+}
+
+func (p *Parser) ParseModulePage(html string, docURL string) (*Module, error) {
+	doc, err := goquery.NewDocumentFromReader(strings.NewReader(html))
+	if err != nil {
+		return nil, err
+	}
+
+	module := &Module{
+		DocURL:    docURL,
+		FetchedAt: time.Now(),
+	}
+
+	module.Name = p.extractModuleName(doc)
+	module.Doc = p.extractModuleDoc(doc)
+	module.Version = p.extractVersion(doc)
+	module.Classes = p.extractClasses(doc, module.Name, docURL)
+	module.Properties = p.extractProperties(doc, docURL)
+	module.Guides = p.extractGuides(doc, docURL)
+
+	return module, nil
+}
+
+func (p *Parser) ParseSearchResults(html string) ([]*SearchResult, error) {
+	doc, err := goquery.NewDocumentFromReader(strings.NewReader(html))
+	if err != nil {
+		return nil, err
+	}
+
+	var results []*SearchResult
+
+	doc.Find(".search-result, .ais-Hits-item, article").Each(func(i int, s *goquery.Selection) {
+		result := &SearchResult{}
+
+		link := s.Find("a").First()
+		result.Name = strings.TrimSpace(link.Text())
+
+		if href, exists := link.Attr("href"); exists {
+			result.DocURL = resolveURL(p.baseURL, href)
+		}
+
+		result.Doc = strings.TrimSpace(s.Find(".summary, .description, p").First().Text())
+
+		if strings.Contains(result.DocURL, "/api/") {
+			result.Kind = "class"
+		} else if strings.Contains(result.DocURL, "/guides/") || strings.Contains(result.DocURL, "/tutorial/") {
+			result.Kind = "guide"
+		} else {
+			result.Kind = "doc"
+		}
+
+		results = append(results, result)
+	})
+
+	return results, nil
+}
+
+func (p *Parser) extractModuleName(doc *goquery.Document) string {
+	title := doc.Find("h1, .title, .page-title").First().Text()
+	title = strings.TrimSpace(title)
+
+	if title != "" {
+		if idx := strings.Index(title, " "); idx > 0 {
+			return title[:idx]
+		}
+		return title
+	}
+
+	return ""
+}
+
+func (p *Parser) extractModuleDoc(doc *goquery.Document) string {
+	docblock := doc.Find(".paragraph:first-child p, .lead, #content p:first-of-type").First()
+	return strings.TrimSpace(docblock.Text())
+}
+
+func (p *Parser) extractVersion(doc *goquery.Document) string {
+	versionEl := doc.Find(".version, .doc-version, [data-version]")
+	return strings.TrimSpace(versionEl.Text())
+}
+
+func (p *Parser) extractClasses(doc *goquery.Document, moduleName string, docURL string) []*Class {
+	var classes []*Class
+
+	doc.Find("table.table tbody tr, .api-list a, .class-link").Each(func(_ int, s *goquery.Selection) {
+		class := &Class{}
+
+		link := s.Find("a")
+		if link.Length() == 0 {
+			link = s
+		}
+
+		class.Name = strings.TrimSpace(link.Text())
+
+		if href, exists := link.Attr("href"); exists {
+			class.DocURL = resolveURL(docURL, href)
+			if strings.Contains(href, "/api/") {
+				class.QualifiedName = extractQualifiedName(href)
+			}
+		}
+
+		class.Doc = strings.TrimSpace(s.Find(".description, td:last-child").Text())
+
+		if class.Name != "" {
+			classes = append(classes, class)
+		}
+	})
+
+	return classes
+}
+
+func (p *Parser) extractProperties(doc *goquery.Document, docURL string) []*Property {
+	var properties []*Property
+
+	doc.Find(".configuration-property, table.properties tbody tr, .config-props dt").Each(func(_ int, s *goquery.Selection) {
+		prop := &Property{}
+
+		nameEl := s.Find(".property-name, code, strong, td:first-child").First()
+		prop.Name = strings.TrimSpace(nameEl.Text())
+
+		prop.Type = strings.TrimSpace(s.Find(".property-type, .type").Text())
+		prop.Default = strings.TrimSpace(s.Find(".default-value, .default").Text())
+		prop.Doc = strings.TrimSpace(s.Find(".description, dd, td:last-child").Text())
+
+		if prop.Name != "" {
+			properties = append(properties, prop)
+		}
+	})
+
+	return properties
+}
+
+func (p *Parser) extractGuides(doc *goquery.Document, docURL string) []*Guide {
+	var guides []*Guide
+
+	doc.Find(".guide-link, .tutorial-link, .guide-card").Each(func(_ int, s *goquery.Selection) {
+		guide := &Guide{}
+
+		link := s.Find("a")
+		if link.Length() == 0 {
+			link = s
+		}
+
+		guide.Title = strings.TrimSpace(link.Text())
+
+		if href, exists := link.Attr("href"); exists {
+			guide.DocURL = resolveURL(docURL, href)
+		}
+
+		guide.Description = strings.TrimSpace(s.Find(".summary, .description").Text())
+
+		if guide.Title != "" {
+			guides = append(guides, guide)
+		}
+	})
+
+	return guides
+}
+
+func (p *Parser) ParseClassPage(html string, docURL string) (*Class, error) {
+	doc, err := goquery.NewDocumentFromReader(strings.NewReader(html))
+	if err != nil {
+		return nil, err
+	}
+
+	class := &Class{
+		DocURL: docURL,
+	}
+
+	header := doc.Find("h1, .title, .class-name").First()
+	class.Name = strings.TrimSpace(header.Text())
+	class.QualifiedName = class.Name
+
+	class.Doc = strings.TrimSpace(doc.Find(".class-description, .javadoc, .class-comment").First().Text())
+
+	class.Methods = p.extractClassMethods(doc, class.Name, docURL)
+	class.Fields = p.extractClassFields(doc, class.Name, docURL)
+	class.Constructors = p.extractClassConstructors(doc, class.Name, docURL)
+
+	return class, nil
+}
+
+func (p *Parser) extractClassMethods(doc *goquery.Document, className string, docURL string) []*Method {
+	var methods []*Method
+
+	doc.Find("table.method-summary tbody tr, .method, .member").Each(func(_ int, s *goquery.Selection) {
+		method := &Method{
+			IsConstructor: false,
+		}
+
+		link := s.Find("a").First()
+		method.Name = strings.TrimSpace(link.Text())
+
+		if method.Name == "" {
+			sig := s.Find(".method-signature, code").Text()
+			method.Name = extractSpringMethodName(sig)
+		}
+
+		method.Signature = strings.TrimSpace(s.Find(".method-signature, code").Text())
+		method.Doc = strings.TrimSpace(s.Find(".method-description, td:last-child, dd").Text())
+
+		if href, exists := link.Attr("href"); exists {
+			if strings.HasPrefix(href, "#") {
+				method.DocURL = docURL + href
+			} else {
+				method.DocURL = resolveURL(docURL, href)
+			}
+			method.QualifiedName = className + "." + method.Name
+		}
+
+		if method.Name != "" {
+			methods = append(methods, method)
+		}
+	})
+
+	return methods
+}
+
+func (p *Parser) extractClassFields(doc *goquery.Document, className string, docURL string) []*Field {
+	var fields []*Field
+
+	doc.Find("table.field-summary tbody tr, .field").Each(func(_ int, s *goquery.Selection) {
+		field := &Field{}
+
+		field.Name = strings.TrimSpace(s.Find(".field-name, a, td:first-child").Text())
+		field.Type = strings.TrimSpace(s.Find(".field-type, td:nth-child(2)").Text())
+		field.Doc = strings.TrimSpace(s.Find(".field-description, td:last-child").Text())
+
+		if field.Name != "" {
+			fields = append(fields, field)
+		}
+	})
+
+	return fields
+}
+
+func (p *Parser) extractClassConstructors(doc *goquery.Document, className string, docURL string) []*Method {
+	var constructors []*Method
+
+	doc.Find("table.constructor-summary tbody tr, .constructor").Each(func(_ int, s *goquery.Selection) {
+		ctor := &Method{
+			IsConstructor: true,
+			Name:          className,
+		}
+
+		ctor.Signature = strings.TrimSpace(s.Find(".constructor-signature, code").Text())
+		ctor.Doc = strings.TrimSpace(s.Find(".constructor-description, td:last-child").Text())
+
+		constructors = append(constructors, ctor)
+	})
+
+	return constructors
+}
+
+func extractSpringMethodName(sig string) string {
+	sig = strings.TrimSpace(sig)
+	if idx := strings.Index(sig, "("); idx > 0 {
+		prefix := sig[:idx]
+		parts := strings.Fields(prefix)
+		if len(parts) > 0 {
+			return parts[len(parts)-1]
+		}
+	}
+	return ""
+}
+
+func extractQualifiedName(href string) string {
+	href = strings.TrimSuffix(href, "/")
+	parts := strings.Split(href, "/")
+	if len(parts) >= 2 {
+		return parts[len(parts)-1]
+	}
+	return ""
+}
+
+func resolveURL(base string, href string) string {
+	if strings.HasPrefix(href, "http") {
+		return href
+	}
+
+	baseURL, err := url.Parse(base)
+	if err != nil {
+		return href
+	}
+
+	hrefURL, err := url.Parse(href)
+	if err != nil {
+		return href
+	}
+
+	return baseURL.ResolveReference(hrefURL).String()
+}

pkg/springdocs/parser_test.go

@@ -0,0 +1,103 @@
+package springdocs
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/PuerkitoBio/goquery"
+)
+
+const testModulePageHTML = `
+<!DOCTYPE html>
+<html>
+<body>
+<h1>Spring Boot Reference</h1>
+<p class="lead">Spring Boot makes it easy to create stand-alone, production-grade Spring based Applications.</p>
+
+<table class="table">
+<tbody>
+<tr>
+<td><a href="/spring-boot/docs/current/api/org/springframework/boot/SpringApplication.html">SpringApplication</a></td>
+<td>Class used to bootstrap and launch a Spring application.</td>
+</tr>
+</tbody>
+</table>
+
+<div class="configuration-property">
+<dt><code>server.port</code></dt>
+<dd>Server HTTP port.</dd>
+</div>
+</body>
+</html>
+`
+
+func TestParseModulePage(t *testing.T) {
+	parser := NewParser()
+	module, err := parser.ParseModulePage(testModulePageHTML, "https://docs.spring.io/spring-boot/docs/current/reference/html/")
+	if err != nil {
+		t.Fatalf("ParseModulePage failed: %v", err)
+	}
+
+	if module.Name == "" {
+		t.Error("Expected non-empty module name")
+	}
+
+	if module.Doc == "" {
+		t.Error("Expected non-empty doc")
+	}
+}
+
+func TestExtractClasses(t *testing.T) {
+	parser := NewParser()
+
+	doc, err := goquery.NewDocumentFromReader(strings.NewReader(testModulePageHTML))
+	if err != nil {
+		t.Fatalf("Failed to parse HTML: %v", err)
+	}
+
+	classes := parser.extractClasses(doc, "spring-boot", "https://docs.spring.io/test")
+
+	if len(classes) == 0 {
+		t.Fatal("Expected at least one class")
+	}
+
+	first := classes[0]
+	if first.Name == "" {
+		t.Error("Expected non-empty class name")
+	}
+}
+
+func TestExtractProperties(t *testing.T) {
+	parser := NewParser()
+
+	doc, err := goquery.NewDocumentFromReader(strings.NewReader(testModulePageHTML))
+	if err != nil {
+		t.Fatalf("Failed to parse HTML: %v", err)
+	}
+
+	props := parser.extractProperties(doc, "https://docs.spring.io/test")
+
+	if len(props) == 0 {
+		t.Skip("No properties extracted from test HTML")
+	}
+}
+
+func TestResolveURL(t *testing.T) {
+	tests := []struct {
+		base     string
+		href     string
+		expected string
+	}{
+		{"https://docs.spring.io", "/api/TestClass.html", "https://docs.spring.io/api/TestClass.html"},
+		{"https://docs.spring.io", "https://example.com/page", "https://example.com/page"},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.href, func(t *testing.T) {
+			got := resolveURL(tt.base, tt.href)
+			if got != tt.expected {
+				t.Errorf("resolveURL(%q, %q) = %q, want %q", tt.base, tt.href, got, tt.expected)
+			}
+		})
+	}
+}

pkg/springdocs/types.go

@@ -0,0 +1,102 @@
+// Package springdocs provides parsing and extraction for Spring Boot documentation
+// from docs.spring.io.
+package springdocs
+
+import "time"
+
+// Module represents a Spring module's documentation.
+type Module struct {
+	Name       string      `json:"name"`
+	Doc        string      `json:"doc,omitempty"`
+	Version    string      `json:"version,omitempty"`
+	DocURL     string      `json:"doc_url"`
+	Packages   []*Package  `json:"packages,omitempty"`
+	Classes    []*Class    `json:"classes,omitempty"`
+	Guides     []*Guide    `json:"guides,omitempty"`
+	Properties []*Property `json:"properties,omitempty"`
+	FetchedAt  time.Time   `json:"fetched_at"`
+}
+
+// Package represents a Spring package.
+type Package struct {
+	Name    string   `json:"name"`
+	Doc     string   `json:"doc,omitempty"`
+	Classes []*Class `json:"classes,omitempty"`
+	DocURL  string   `json:"doc_url"`
+}
+
+// Class represents a Spring class.
+type Class struct {
+	QualifiedName string    `json:"qualified_name"`
+	Name          string    `json:"name"`
+	Package       string    `json:"package"`
+	Kind          string    `json:"kind"`
+	Doc           string    `json:"doc,omitempty"`
+	Annotations   []string  `json:"annotations,omitempty"`
+	SuperClass    string    `json:"super_class,omitempty"`
+	Interfaces    []string  `json:"interfaces,omitempty"`
+	Fields        []*Field  `json:"fields,omitempty"`
+	Methods       []*Method `json:"methods,omitempty"`
+	Constructors  []*Method `json:"constructors,omitempty"`
+	DocURL        string    `json:"doc_url"`
+	Since         string    `json:"since,omitempty"`
+}
+
+// Field represents a class field.
+type Field struct {
+	Name        string   `json:"name"`
+	Type        string   `json:"type"`
+	Doc         string   `json:"doc,omitempty"`
+	Annotations []string `json:"annotations,omitempty"`
+	DocURL      string   `json:"doc_url"`
+}
+
+// Method represents a method.
+type Method struct {
+	Name          string       `json:"name"`
+	Doc           string       `json:"doc,omitempty"`
+	Annotations   []string     `json:"annotations,omitempty"`
+	ReturnType    string       `json:"return_type,omitempty"`
+	Parameters    []*Parameter `json:"parameters,omitempty"`
+	Signature     string       `json:"signature"`
+	QualifiedName string       `json:"qualified_name"`
+	DocURL        string       `json:"doc_url"`
+	IsConstructor bool         `json:"is_constructor"`
+}
+
+// Parameter represents a method parameter.
+type Parameter struct {
+	Name string `json:"name"`
+	Type string `json:"type"`
+	Doc  string `json:"doc,omitempty"`
+}
+
+// Guide represents a Spring guide/tutorial.
+type Guide struct {
+	Title       string   `json:"title"`
+	Description string   `json:"description,omitempty"`
+	DocURL      string   `json:"doc_url"`
+	Tags        []string `json:"tags,omitempty"`
+	Level       string   `json:"level,omitempty"` // beginner, intermediate, advanced
+}
+
+// Property represents a Spring configuration property.
+type Property struct {
+	Name       string `json:"name"`
+	Value      string `json:"value,omitempty"`
+	Type       string `json:"type,omitempty"`
+	Doc        string `json:"doc,omitempty"`
+	Default    string `json:"default,omitempty"`
+	DocURL     string `json:"doc_url"`
+	Deprecated string `json:"deprecated,omitempty"`
+}
+
+// SearchResult represents a search result.
+type SearchResult struct {
+	Name   string `json:"name"`
+	Kind   string `json:"kind"` // class, method, property, guide
+	Module string `json:"module,omitempty"`
+	Doc    string `json:"doc,omitempty"`
+	DocURL string `json:"doc_url"`
+	Score  int    `json:"score"`
+}

pkg/tsdocs/parser.go

@@ -0,0 +1,307 @@
+package tsdocs
+
+import (
+	"net/url"
+	"strings"
+	"time"
+
+	"github.com/PuerkitoBio/goquery"
+)
+
+type Parser struct {
+	baseURL string
+}
+
+func NewParser() *Parser {
+	return &Parser{
+		baseURL: "https://www.typescriptlang.org",
+	}
+}
+
+func (p *Parser) ParseModulePage(html string, docURL string) (*Module, error) {
+	doc, err := goquery.NewDocumentFromReader(strings.NewReader(html))
+	if err != nil {
+		return nil, err
+	}
+
+	module := &Module{
+		DocURL:    docURL,
+		FetchedAt: time.Now(),
+	}
+
+	module.Name = p.extractModuleName(doc)
+	module.Doc = p.extractModuleDoc(doc)
+	module.Interfaces = p.extractInterfaces(doc, module.Name, docURL)
+	module.Types = p.extractTypeAliases(doc, module.Name, docURL)
+	module.Functions = p.extractFunctions(doc, module.Name, docURL)
+	module.Classes = p.extractClasses(doc, module.Name, docURL)
+	module.Enums = p.extractEnums(doc, module.Name, docURL)
+	module.Variables = p.extractVariables(doc, module.Name, docURL)
+
+	return module, nil
+}
+
+func (p *Parser) ParseSearchResults(html string) ([]*SearchResult, error) {
+	doc, err := goquery.NewDocumentFromReader(strings.NewReader(html))
+	if err != nil {
+		return nil, err
+	}
+
+	var results []*SearchResult
+
+	doc.Find(".search-result, .ais-Hits-item, li.result").Each(func(i int, s *goquery.Selection) {
+		result := &SearchResult{}
+
+		link := s.Find("a").First()
+		result.Name = strings.TrimSpace(link.Text())
+
+		if href, exists := link.Attr("href"); exists {
+			result.DocURL = resolveURL(p.baseURL, href)
+		}
+
+		result.Doc = strings.TrimSpace(s.Find(".summary, p, .description").First().Text())
+
+		if s.HasClass("interface") || strings.Contains(s.Text(), "interface") {
+			result.Kind = "interface"
+		} else if s.HasClass("type") || strings.Contains(s.Text(), "type") {
+			result.Kind = "type"
+		} else if s.HasClass("function") || strings.Contains(s.Text(), "function") {
+			result.Kind = "function"
+		} else if s.HasClass("class") || strings.Contains(s.Text(), "class") {
+			result.Kind = "class"
+		} else {
+			result.Kind = "doc"
+		}
+
+		results = append(results, result)
+	})
+
+	return results, nil
+}
+
+func (p *Parser) extractModuleName(doc *goquery.Document) string {
+	title := doc.Find("h1, .title, .page-title").First().Text()
+	return strings.TrimSpace(title)
+}
+
+func (p *Parser) extractModuleDoc(doc *goquery.Document) string {
+	docblock := doc.Find(".markdown p:first-of-type, .content p:first-of-type, #main p").First()
+	return strings.TrimSpace(docblock.Text())
+}
+
+func (p *Parser) extractInterfaces(doc *goquery.Document, moduleName string, docURL string) []*Interface {
+	var interfaces []*Interface
+
+	doc.Find("h2, h3, .context-item, .api-item").Each(func(_ int, s *goquery.Selection) {
+		text := s.Text()
+		if !strings.Contains(strings.ToLower(text), "interface") {
+			return
+		}
+
+		iface := &Interface{}
+
+		nameEl := s.Find("code, .name, a").First()
+		if nameEl.Length() == 0 {
+			nameEl = s
+		}
+		iface.Name = strings.TrimSpace(nameEl.Text())
+		iface.Name = strings.TrimSuffix(iface.Name, "<")
+		iface.Name = strings.Split(iface.Name, "<")[0]
+		iface.Name = strings.TrimSpace(iface.Name)
+
+		if id, exists := s.Attr("id"); exists {
+			iface.DocURL = docURL + "#" + id
+		}
+
+		next := s.Next()
+		for next.Length() > 0 && !next.Is("h2, h3") {
+			if next.Is("p") && iface.Doc == "" {
+				iface.Doc = strings.TrimSpace(next.Text())
+			}
+			next = next.Next()
+		}
+
+		if iface.Name != "" && iface.Name != "interface" {
+			interfaces = append(interfaces, iface)
+		}
+	})
+
+	return interfaces
+}
+
+func (p *Parser) extractTypeAliases(doc *goquery.Document, moduleName string, docURL string) []*TypeAlias {
+	var types []*TypeAlias
+
+	doc.Find("h2, h3, .context-item, .api-item").Each(func(_ int, s *goquery.Selection) {
+		text := s.Text()
+		if !strings.Contains(strings.ToLower(text), "type") {
+			return
+		}
+
+		ta := &TypeAlias{}
+
+		nameEl := s.Find("code, .name").First()
+		if nameEl.Length() == 0 {
+			nameEl = s
+		}
+		ta.Name = strings.TrimSpace(nameEl.Text())
+		ta.Name = strings.TrimSuffix(ta.Name, "<")
+		ta.Name = strings.Split(ta.Name, "<")[0]
+		ta.Name = strings.TrimSpace(ta.Name)
+
+		if id, exists := s.Attr("id"); exists {
+			ta.DocURL = docURL + "#" + id
+		}
+
+		if ta.Name != "" && ta.Name != "type" {
+			types = append(types, ta)
+		}
+	})
+
+	return types
+}
+
+func (p *Parser) extractFunctions(doc *goquery.Document, moduleName string, docURL string) []*Function {
+	var functions []*Function
+
+	doc.Find("h2, h3, .context-item, .api-item, pre code").Each(func(_ int, s *goquery.Selection) {
+		text := s.Text()
+		if !strings.Contains(text, "function") && !strings.Contains(text, "(") {
+			return
+		}
+
+		fn := &Function{}
+
+		sigText := text
+		if idx := strings.Index(sigText, "("); idx > 0 {
+			prefix := sigText[:idx]
+			parts := strings.Fields(prefix)
+			if len(parts) > 0 {
+				fn.Name = parts[len(parts)-1]
+			}
+			fn.Signature = strings.TrimSpace(sigText)
+		}
+
+		if id, exists := s.Attr("id"); exists {
+			fn.DocURL = docURL + "#" + id
+		}
+
+		next := s.Next()
+		for next.Length() > 0 && !next.Is("h2, h3, pre") {
+			if next.Is("p") && fn.Doc == "" {
+				fn.Doc = strings.TrimSpace(next.Text())
+			}
+			next = next.Next()
+		}
+
+		if fn.Name != "" {
+			functions = append(functions, fn)
+		}
+	})
+
+	return functions
+}
+
+func (p *Parser) extractClasses(doc *goquery.Document, moduleName string, docURL string) []*Class {
+	var classes []*Class
+
+	doc.Find("h2, h3, .context-item, .api-item").Each(func(_ int, s *goquery.Selection) {
+		text := s.Text()
+		if !strings.Contains(strings.ToLower(text), "class") {
+			return
+		}
+
+		class := &Class{}
+
+		nameEl := s.Find("code, .name").First()
+		if nameEl.Length() == 0 {
+			nameEl = s
+		}
+		class.Name = strings.TrimSpace(nameEl.Text())
+		class.Name = strings.TrimSuffix(class.Name, "<")
+		class.Name = strings.Split(class.Name, "<")[0]
+		class.Name = strings.TrimSpace(class.Name)
+
+		if id, exists := s.Attr("id"); exists {
+			class.DocURL = docURL + "#" + id
+		}
+
+		if class.Name != "" && class.Name != "class" {
+			classes = append(classes, class)
+		}
+	})
+
+	return classes
+}
+
+func (p *Parser) extractEnums(doc *goquery.Document, moduleName string, docURL string) []*Enum {
+	var enums []*Enum
+
+	doc.Find("h2, h3, .context-item").Each(func(_ int, s *goquery.Selection) {
+		text := s.Text()
+		if !strings.Contains(strings.ToLower(text), "enum") {
+			return
+		}
+
+		enum := &Enum{}
+
+		nameEl := s.Find("code, .name").First()
+		if nameEl.Length() == 0 {
+			nameEl = s
+		}
+		enum.Name = strings.TrimSpace(nameEl.Text())
+
+		if id, exists := s.Attr("id"); exists {
+			enum.DocURL = docURL + "#" + id
+		}
+
+		if enum.Name != "" && enum.Name != "enum" {
+			enums = append(enums, enum)
+		}
+	})
+
+	return enums
+}
+
+func (p *Parser) extractVariables(doc *goquery.Document, moduleName string, docURL string) []*Variable {
+	var variables []*Variable
+
+	doc.Find("pre code").Each(func(_ int, s *goquery.Selection) {
+		text := s.Text()
+		if strings.Contains(text, "const ") || strings.Contains(text, "let ") || strings.Contains(text, "var ") {
+			v := &Variable{}
+
+			if idx := strings.Index(text, "="); idx > 0 {
+				decl := text[:idx]
+				decl = strings.TrimPrefix(decl, "const")
+				decl = strings.TrimPrefix(decl, "let")
+				decl = strings.TrimPrefix(decl, "var")
+				v.Name = strings.TrimSpace(decl)
+			}
+
+			if v.Name != "" {
+				variables = append(variables, v)
+			}
+		}
+	})
+
+	return variables
+}
+
+func resolveURL(base string, href string) string {
+	if strings.HasPrefix(href, "http") {
+		return href
+	}
+
+	baseURL, err := url.Parse(base)
+	if err != nil {
+		return href
+	}
+
+	hrefURL, err := url.Parse(href)
+	if err != nil {
+		return href
+	}
+
+	return baseURL.ResolveReference(hrefURL).String()
+}

pkg/tsdocs/parser_test.go

@@ -0,0 +1,97 @@
+package tsdocs
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/PuerkitoBio/goquery"
+)
+
+const testModulePageHTML = `
+<!DOCTYPE html>
+<html>
+<body>
+<h1>TypeScript Handbook</h1>
+<p class="content">TypeScript is a strongly typed programming language that builds on JavaScript.</p>
+
+<h2 id="string">interface String</h2>
+<p>Allows manipulation and formatting of text strings.</p>
+
+<h3 id="concat">concat(...strings: string[]): string</h3>
+<p>Returns a string that contains the concatenation of two or more strings.</p>
+
+<h2 id="Array">interface Array&lt;T&gt;</h2>
+<p>An array of values.</p>
+
+<pre><code>function identity<T>(arg: T): T {
+  return arg;
+}</code></pre>
+</body>
+</html>
+`
+
+func TestParseModulePage(t *testing.T) {
+	parser := NewParser()
+	module, err := parser.ParseModulePage(testModulePageHTML, "https://www.typescriptlang.org/docs/handbook/")
+	if err != nil {
+		t.Fatalf("ParseModulePage failed: %v", err)
+	}
+
+	if module.Name == "" {
+		t.Error("Expected non-empty module name")
+	}
+}
+
+func TestExtractInterfaces(t *testing.T) {
+	parser := NewParser()
+
+	doc, err := goquery.NewDocumentFromReader(strings.NewReader(testModulePageHTML))
+	if err != nil {
+		t.Fatalf("Failed to parse HTML: %v", err)
+	}
+
+	interfaces := parser.extractInterfaces(doc, "typescript", "https://www.typescriptlang.org/docs/test")
+
+	for _, iface := range interfaces {
+		if iface.Name == "interface" {
+			t.Error("Should not include 'interface' as a name")
+		}
+	}
+}
+
+func TestExtractFunctions(t *testing.T) {
+	parser := NewParser()
+
+	doc, err := goquery.NewDocumentFromReader(strings.NewReader(testModulePageHTML))
+	if err != nil {
+		t.Fatalf("Failed to parse HTML: %v", err)
+	}
+
+	functions := parser.extractFunctions(doc, "typescript", "https://www.typescriptlang.org/docs/test")
+
+	for _, fn := range functions {
+		if fn.Name != "" && fn.Signature != "" {
+			return
+		}
+	}
+}
+
+func TestResolveURL(t *testing.T) {
+	tests := []struct {
+		base     string
+		href     string
+		expected string
+	}{
+		{"https://www.typescriptlang.org", "/docs/handbook", "https://www.typescriptlang.org/docs/handbook"},
+		{"https://www.typescriptlang.org", "https://example.com/page", "https://example.com/page"},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.href, func(t *testing.T) {
+			got := resolveURL(tt.base, tt.href)
+			if got != tt.expected {
+				t.Errorf("resolveURL(%q, %q) = %q, want %q", tt.base, tt.href, got, tt.expected)
+			}
+		})
+	}
+}

pkg/tsdocs/types.go

@@ -0,0 +1,177 @@
+// Package tsdocs provides parsing and extraction for TypeScript documentation
+// from typescriptlang.org.
+package tsdocs
+
+import "time"
+
+// Module represents a TypeScript module's documentation.
+type Module struct {
+	Name       string       `json:"name"`
+	Doc        string       `json:"doc,omitempty"`
+	Version    string       `json:"version,omitempty"`
+	DocURL     string       `json:"doc_url"`
+	Interfaces []*Interface `json:"interfaces,omitempty"`
+	Types      []*TypeAlias `json:"types,omitempty"`
+	Functions  []*Function  `json:"functions,omitempty"`
+	Classes    []*Class     `json:"classes,omitempty"`
+	Variables  []*Variable  `json:"variables,omitempty"`
+	Enums      []*Enum      `json:"enums,omitempty"`
+	Namespaces []*Namespace `json:"namespaces,omitempty"`
+	FetchedAt  time.Time    `json:"fetched_at"`
+}
+
+// Interface represents a TypeScript interface.
+type Interface struct {
+	Name            string       `json:"name"`
+	Doc             string       `json:"doc,omitempty"`
+	Modifiers       []string     `json:"modifiers,omitempty"`
+	Extends         []string     `json:"extends,omitempty"`
+	Properties      []*Property  `json:"properties,omitempty"`
+	Methods         []*Method    `json:"methods,omitempty"`
+	CallSignatures  []*Signature `json:"call_signatures,omitempty"`
+	IndexSignatures []*Signature `json:"index_signatures,omitempty"`
+	TypeParams      []*TypeParam `json:"type_params,omitempty"`
+	DocURL          string       `json:"doc_url"`
+	Deprecated      string       `json:"deprecated,omitempty"`
+}
+
+// TypeAlias represents a TypeScript type alias.
+type TypeAlias struct {
+	Name       string       `json:"name"`
+	Doc        string       `json:"doc,omitempty"`
+	Type       string       `json:"type"`
+	TypeParams []*TypeParam `json:"type_params,omitempty"`
+	DocURL     string       `json:"doc_url"`
+	Deprecated string       `json:"deprecated,omitempty"`
+}
+
+// Class represents a TypeScript class.
+type Class struct {
+	Name         string       `json:"name"`
+	Doc          string       `json:"doc,omitempty"`
+	Modifiers    []string     `json:"modifiers,omitempty"`
+	Extends      string       `json:"extends,omitempty"`
+	Implements   []string     `json:"implements,omitempty"`
+	Properties   []*Property  `json:"properties,omitempty"`
+	Methods      []*Method    `json:"methods,omitempty"`
+	Constructors []*Method    `json:"constructors,omitempty"`
+	TypeParams   []*TypeParam `json:"type_params,omitempty"`
+	DocURL       string       `json:"doc_url"`
+	Deprecated   string       `json:"deprecated,omitempty"`
+}
+
+// Property represents a class or interface property.
+type Property struct {
+	Name       string   `json:"name"`
+	Type       string   `json:"type"`
+	Doc        string   `json:"doc,omitempty"`
+	Modifiers  []string `json:"modifiers,omitempty"`
+	Optional   bool     `json:"optional"`
+	ReadOnly   bool     `json:"read_only"`
+	Default    string   `json:"default,omitempty"`
+	DocURL     string   `json:"doc_url"`
+	Deprecated string   `json:"deprecated,omitempty"`
+}
+
+// Method represents a class or interface method.
+type Method struct {
+	Name       string       `json:"name"`
+	Doc        string       `json:"doc,omitempty"`
+	Modifiers  []string     `json:"modifiers,omitempty"`
+	Parameters []*Parameter `json:"parameters,omitempty"`
+	ReturnType string       `json:"return_type,omitempty"`
+	TypeParams []*TypeParam `json:"type_params,omitempty"`
+	Signature  string       `json:"signature"`
+	DocURL     string       `json:"doc_url"`
+	Deprecated string       `json:"deprecated,omitempty"`
+	IsStatic   bool         `json:"is_static"`
+	IsAbstract bool         `json:"is_abstract"`
+}
+
+// Function represents a standalone function.
+type Function struct {
+	Name       string       `json:"name"`
+	Doc        string       `json:"doc,omitempty"`
+	Signature  string       `json:"signature,omitempty"`
+	Parameters []*Parameter `json:"parameters,omitempty"`
+	ReturnType string       `json:"return_type,omitempty"`
+	TypeParams []*TypeParam `json:"type_params,omitempty"`
+	Signatures []*Signature `json:"signatures,omitempty"`
+	DocURL     string       `json:"doc_url"`
+	Deprecated string       `json:"deprecated,omitempty"`
+}
+
+// Signature represents a function/method signature.
+type Signature struct {
+	Parameters []*Parameter `json:"parameters,omitempty"`
+	ReturnType string       `json:"return_type,omitempty"`
+	Doc        string       `json:"doc,omitempty"`
+}
+
+// Parameter represents a function/method parameter.
+type Parameter struct {
+	Name     string `json:"name"`
+	Type     string `json:"type"`
+	Doc      string `json:"doc,omitempty"`
+	Optional bool   `json:"optional"`
+	Default  string `json:"default,omitempty"`
+	Rest     bool   `json:"rest"`
+}
+
+// TypeParam represents a type parameter (generic).
+type TypeParam struct {
+	Name       string `json:"name"`
+	Constraint string `json:"constraint,omitempty"`
+	Default    string `json:"default,omitempty"`
+	Doc        string `json:"doc,omitempty"`
+}
+
+// Variable represents a const/let/var declaration.
+type Variable struct {
+	Name       string   `json:"name"`
+	Type       string   `json:"type,omitempty"`
+	Value      string   `json:"value,omitempty"`
+	Doc        string   `json:"doc,omitempty"`
+	Modifiers  []string `json:"modifiers,omitempty"`
+	DocURL     string   `json:"doc_url"`
+	Deprecated string   `json:"deprecated,omitempty"`
+}
+
+// Enum represents a TypeScript enum.
+type Enum struct {
+	Name       string        `json:"name"`
+	Doc        string        `json:"doc,omitempty"`
+	Modifiers  []string      `json:"modifiers,omitempty"`
+	Members    []*EnumMember `json:"members,omitempty"`
+	DocURL     string        `json:"doc_url"`
+	Deprecated string        `json:"deprecated,omitempty"`
+}
+
+// EnumMember represents an enum member.
+type EnumMember struct {
+	Name  string `json:"name"`
+	Value string `json:"value,omitempty"`
+	Doc   string `json:"doc,omitempty"`
+}
+
+// Namespace represents a TypeScript namespace.
+type Namespace struct {
+	Name       string       `json:"name"`
+	Doc        string       `json:"doc,omitempty"`
+	Interfaces []*Interface `json:"interfaces,omitempty"`
+	Types      []*TypeAlias `json:"types,omitempty"`
+	Functions  []*Function  `json:"functions,omitempty"`
+	Classes    []*Class     `json:"classes,omitempty"`
+	Variables  []*Variable  `json:"variables,omitempty"`
+	DocURL     string       `json:"doc_url"`
+}
+
+// SearchResult represents a search result.
+type SearchResult struct {
+	Name   string `json:"name"`
+	Kind   string `json:"kind"` // interface, type, function, class, variable, enum
+	Module string `json:"module,omitempty"`
+	Doc    string `json:"doc,omitempty"`
+	DocURL string `json:"doc_url"`
+	Score  int    `json:"score"`
+}

pkg/types/types.go

@@ -0,0 +1,71 @@
+// Package types provides shared types and interfaces.
+package types
+
+import "time"
+
+// Document represents an indexed document.
+type Document struct {
+	ID        string                 `json:"id"`
+	Source    string                 `json:"source"`
+	Type      string                 `json:"type"`
+	Title     string                 `json:"title"`
+	Content   string                 `json:"content"`
+	URL       string                 `json:"url,omitempty"`
+	Metadata  map[string]interface{} `json:"metadata,omitempty"`
+	Hash      string                 `json:"hash"`
+	Timestamp time.Time              `json:"timestamp"`
+}
+
+// Chunk represents a text chunk for indexing.
+type Chunk struct {
+	ID       string         `json:"id"`
+	DocID    string         `json:"doc_id"`
+	Content  string         `json:"content"`
+	Vector   []float32      `json:"vector,omitempty"`
+	Metadata map[string]any `json:"metadata,omitempty"`
+	Position int            `json:"position"`
+}
+
+// SearchResult represents a search result.
+type SearchResult struct {
+	ID       string         `json:"id"`
+	DocID    string         `json:"doc_id"`
+	Content  string         `json:"content"`
+	Score    float64        `json:"score"`
+	Source   string         `json:"source"`
+	Metadata map[string]any `json:"metadata,omitempty"`
+}
+
+// Source represents a documentation source.
+type Source struct {
+	Name     string   `yaml:"name"`
+	Type     string   `yaml:"type"`
+	URL      string   `yaml:"url,omitempty"`
+	Repo     string   `yaml:"repo,omitempty"`
+	Branch   string   `yaml:"branch,omitempty"`
+	Path     string   `yaml:"path,omitempty"`
+	Include  []string `yaml:"include,omitempty"`
+	Exclude  []string `yaml:"exclude,omitempty"`
+	Schedule string   `yaml:"schedule,omitempty"`
+}
+
+// Status represents index status.
+type Status struct {
+	Healthy       bool           `json:"healthy"`
+	DocumentCount int            `json:"document_count"`
+	ChunkCount    int            `json:"chunk_count"`
+	Dimension     int            `json:"dimension"`
+	LastUpdated   time.Time      `json:"last_updated"`
+	StorageBytes  int64          `json:"storage_bytes"`
+	Sources       []SourceStatus `json:"sources"`
+}
+
+// SourceStatus represents status of a single source.
+type SourceStatus struct {
+	Name     string    `json:"name"`
+	Type     string    `json:"type"`
+	DocCount int       `json:"doc_count"`
+	LastSync time.Time `json:"last_sync"`
+	Status   string    `json:"status"`
+	NextSync time.Time `json:"next_sync,omitempty"`
+}

pkg/vuedocs/parser.go

@@ -0,0 +1,333 @@
+package vuedocs
+
+import (
+	"net/url"
+	"strings"
+	"time"
+
+	"github.com/PuerkitoBio/goquery"
+)
+
+type Parser struct {
+	baseURL string
+}
+
+func NewParser() *Parser {
+	return &Parser{
+		baseURL: "https://vuejs.org",
+	}
+}
+
+func (p *Parser) ParseReferencePage(html string, docURL string) (*Reference, error) {
+	doc, err := goquery.NewDocumentFromReader(strings.NewReader(html))
+	if err != nil {
+		return nil, err
+	}
+
+	ref := &Reference{
+		DocURL:    docURL,
+		FetchedAt: time.Now(),
+	}
+
+	ref.GlobalAPI = p.extractGlobalAPI(doc, docURL)
+	ref.Composition = p.extractCompositionAPI(doc, docURL)
+	ref.Options = p.extractOptionsAPI(doc, docURL)
+	ref.Directives = p.extractDirectives(doc, docURL)
+	ref.Components = p.extractComponents(doc, docURL)
+	ref.SpecialAttrs = p.extractSpecialAttrs(doc, docURL)
+
+	return ref, nil
+}
+
+func (p *Parser) ParseSearchResults(html string) ([]*SearchResult, error) {
+	doc, err := goquery.NewDocumentFromReader(strings.NewReader(html))
+	if err != nil {
+		return nil, err
+	}
+
+	var results []*SearchResult
+
+	doc.Find(".search-result, a[href*='/api/'], .nav-link, .api-link").Each(func(i int, s *goquery.Selection) {
+		result := &SearchResult{}
+
+		result.Name = strings.TrimSpace(s.Text())
+
+		if href, exists := s.Attr("href"); exists {
+			result.DocURL = resolveURL(p.baseURL, href)
+
+			if strings.Contains(href, "/composition-") {
+				result.Kind = "composition"
+			} else if strings.Contains(href, "/options-") {
+				result.Kind = "options"
+			} else if strings.Contains(href, "/directive") {
+				result.Kind = "directive"
+			} else if strings.Contains(href, "/component") {
+				result.Kind = "component"
+			} else {
+				result.Kind = "api"
+			}
+		}
+
+		if result.Name != "" {
+			results = append(results, result)
+		}
+	})
+
+	return results, nil
+}
+
+func (p *Parser) extractGlobalAPI(doc *goquery.Document, docURL string) []*API {
+	var apis []*API
+
+	doc.Find("h2, h3, .api-item, [id]").Each(func(_ int, s *goquery.Selection) {
+		id, _ := s.Attr("id")
+		text := s.Text()
+
+		globalAPIs := []string{"createApp", "createSSRApp", "defineComponent", "defineAsyncComponent", "resolveComponent", "resolveDirective", "withDirectives", "withKeys", "withModifiers"}
+		isGlobal := false
+		for _, name := range globalAPIs {
+			if id == name || strings.Contains(text, name) {
+				isGlobal = true
+				break
+			}
+		}
+
+		if !isGlobal {
+			return
+		}
+
+		api := &API{}
+
+		nameEl := s.Find("code, .name").First()
+		if nameEl.Length() == 0 {
+			nameEl = s
+		}
+		api.Name = strings.TrimSpace(nameEl.Text())
+		api.Name = strings.TrimSuffix(api.Name, "(")
+
+		api.DocURL = docURL + "#" + api.Name
+
+		next := s.Next()
+		for next.Length() > 0 && !next.Is("h2, h3, h4") {
+			if next.Is("p") && api.Doc == "" {
+				api.Doc = strings.TrimSpace(next.Text())
+			}
+			next = next.Next()
+		}
+
+		if api.Name != "" {
+			apis = append(apis, api)
+		}
+	})
+
+	return apis
+}
+
+func (p *Parser) extractCompositionAPI(doc *goquery.Document, docURL string) []*Composition {
+	var compos []*Composition
+
+	doc.Find("h2, h3, h4, .api-item, [id^='ref'], [id^='reactive'], [id^='computed'], [id^='watch'], [id^='on']").Each(func(_ int, s *goquery.Selection) {
+		id, _ := s.Attr("id")
+		text := s.Text()
+
+		if id == "" && !strings.Contains(text, "(") {
+			return
+		}
+
+		compNames := []string{"ref", "reactive", "computed", "watch", "watchEffect", "watchPostEffect", "watchSyncEffect", "onMounted", "onUpdated", "onUnmounted", "onBeforeMount", "onBeforeUpdate", "onBeforeUnmount", "onErrorCaptured", "onRenderTracked", "onRenderTriggered", "provide", "inject", "toRef", "toRefs", "toValue", "unref", "isRef", "shallowRef", "triggerRef", "customRef", "shallowReactive", "readonly", "shallowReadonly", "markRaw", "toRaw"}
+		isComp := false
+		for _, name := range compNames {
+			if id == name || strings.HasPrefix(id, name+"-") || strings.Contains(text, name+"(") {
+				isComp = true
+				break
+			}
+		}
+
+		if !isComp {
+			return
+		}
+
+		comp := &Composition{}
+
+		nameEl := s.Find("code, .name").First()
+		if nameEl.Length() == 0 {
+			nameEl = s
+		}
+		comp.Name = strings.TrimSpace(nameEl.Text())
+		comp.Name = strings.TrimSuffix(comp.Name, "(")
+
+		if comp.Name == "" && id != "" {
+			comp.Name = id
+		}
+
+		comp.DocURL = docURL + "#" + comp.Name
+
+		next := s.Next()
+		for next.Length() > 0 && !next.Is("h2, h3, h4") {
+			if next.Is("p") && comp.Doc == "" {
+				comp.Doc = strings.TrimSpace(next.Text())
+			}
+			next = next.Next()
+		}
+
+		if comp.Name != "" {
+			compos = append(compos, comp)
+		}
+	})
+
+	return compos
+}
+
+func (p *Parser) extractOptionsAPI(doc *goquery.Document, docURL string) []*Options {
+	var opts []*Options
+
+	optionNames := []string{"data", "props", "computed", "methods", "watch", "emits", "expose", "setup", "name", "components", "directives", "inheritAttrs", "extends", "mixins", "provide", "inject", "template", "render", "renderTracked", "renderTriggered", "errorCaptured", "beforeCreate", "created", "beforeMount", "mounted", "beforeUpdate", "updated", "beforeUnmount", "unmounted", "activated", "deactivated", "serverPrefetch"}
+
+	doc.Find("h2, h3, h4, .api-item").Each(func(_ int, s *goquery.Selection) {
+		id, _ := s.Attr("id")
+		text := s.Text()
+
+		for _, name := range optionNames {
+			if id == name || strings.Contains(strings.ToLower(text), name) {
+				opt := &Options{
+					Name: name,
+				}
+
+				opt.DocURL = docURL + "#" + name
+
+				next := s.Next()
+				for next.Length() > 0 && !next.Is("h2, h3, h4") {
+					if next.Is("p") && opt.Doc == "" {
+						opt.Doc = strings.TrimSpace(next.Text())
+					}
+					next = next.Next()
+				}
+
+				opts = append(opts, opt)
+				break
+			}
+		}
+	})
+
+	return opts
+}
+
+func (p *Parser) extractDirectives(doc *goquery.Document, docURL string) []*Directive {
+	var directives []*Directive
+
+	directiveNames := []string{"v-bind", "v-on", "v-model", "v-if", "v-else", "v-else-if", "v-show", "v-for", "v-slot", "v-text", "v-html", "v-cloak", "v-once", "v-pre", "v-memo"}
+
+	doc.Find("h2, h3, h4, .api-item").Each(func(_ int, s *goquery.Selection) {
+		id, _ := s.Attr("id")
+		text := s.Text()
+
+		for _, name := range directiveNames {
+			if id == name || strings.Contains(text, name) {
+				dir := &Directive{
+					Name: name,
+				}
+
+				dir.DocURL = docURL + "#" + name
+
+				next := s.Next()
+				for next.Length() > 0 && !next.Is("h2, h3, h4") {
+					if next.Is("p") && dir.Doc == "" {
+						dir.Doc = strings.TrimSpace(next.Text())
+					}
+					next = next.Next()
+				}
+
+				directives = append(directives, dir)
+				break
+			}
+		}
+	})
+
+	return directives
+}
+
+func (p *Parser) extractComponents(doc *goquery.Document, docURL string) []*Component {
+	var components []*Component
+
+	componentNames := []string{"Transition", "TransitionGroup", "KeepAlive", "Teleport", "Suspense"}
+
+	doc.Find("h2, h3, h4, .api-item").Each(func(_ int, s *goquery.Selection) {
+		id, _ := s.Attr("id")
+		text := s.Text()
+
+		for _, name := range componentNames {
+			if id == name || strings.Contains(text, name) {
+				comp := &Component{
+					Name: name,
+				}
+
+				comp.DocURL = docURL + "#" + name
+
+				next := s.Next()
+				for next.Length() > 0 && !next.Is("h2, h3, h4") {
+					if next.Is("p") && comp.Doc == "" {
+						comp.Doc = strings.TrimSpace(next.Text())
+					}
+					next = next.Next()
+				}
+
+				components = append(components, comp)
+				break
+			}
+		}
+	})
+
+	return components
+}
+
+func (p *Parser) extractSpecialAttrs(doc *goquery.Document, docURL string) []*SpecialAttr {
+	var attrs []*SpecialAttr
+
+	attrNames := []string{"key", "ref", "is"}
+
+	doc.Find("h2, h3, h4, .api-item").Each(func(_ int, s *goquery.Selection) {
+		id, _ := s.Attr("id")
+		text := s.Text()
+
+		for _, name := range attrNames {
+			if id == name+"-attribute" || id == name || strings.Contains(text, name+" attribute") {
+				attr := &SpecialAttr{
+					Name: name,
+				}
+
+				attr.DocURL = docURL + "#" + name
+
+				next := s.Next()
+				for next.Length() > 0 && !next.Is("h2, h3, h4") {
+					if next.Is("p") && attr.Doc == "" {
+						attr.Doc = strings.TrimSpace(next.Text())
+					}
+					next = next.Next()
+				}
+
+				attrs = append(attrs, attr)
+				break
+			}
+		}
+	})
+
+	return attrs
+}
+
+func resolveURL(base string, href string) string {
+	if strings.HasPrefix(href, "http") {
+		return href
+	}
+
+	baseURL, err := url.Parse(base)
+	if err != nil {
+		return href
+	}
+
+	hrefURL, err := url.Parse(href)
+	if err != nil {
+		return href
+	}
+
+	return baseURL.ResolveReference(hrefURL).String()
+}

pkg/vuedocs/parser_test.go

@@ -0,0 +1,112 @@
+package vuedocs
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/PuerkitoBio/goquery"
+)
+
+const testReferencePageHTML = `
+<!DOCTYPE html>
+<html>
+<body>
+<h1>Vue API Reference</h1>
+
+<h2 id="ref">ref()</h2>
+<p>Takes an inner value and returns a reactive and mutable ref object.</p>
+<pre><code>function ref<T>(value: T): Ref<T></code></pre>
+
+<h2 id="reactive">reactive()</h2>
+<p>Returns a reactive proxy of the object.</p>
+<pre><code>function reactive<T extends object>(target: T): T</code></pre>
+
+<h2 id="computed">computed()</h2>
+<p>Takes a getter function and returns a readonly reactive ref object.</p>
+
+<h3 id="v-bind">v-bind</h3>
+<p>Dynamically binds one or more attributes to an expression.</p>
+
+<h3 id="Transition">Transition</h3>
+<p>Provides animated transition effects when elements enter or leave the DOM.</p>
+</body>
+</html>
+`
+
+func TestParseReferencePage(t *testing.T) {
+	parser := NewParser()
+	ref, err := parser.ParseReferencePage(testReferencePageHTML, "https://vuejs.org/api/")
+	if err != nil {
+		t.Fatalf("ParseReferencePage failed: %v", err)
+	}
+
+	if len(ref.Composition) == 0 {
+		t.Error("Expected at least one composition API item")
+	}
+
+	if len(ref.Directives) == 0 {
+		t.Error("Expected at least one directive")
+	}
+
+	if len(ref.Components) == 0 {
+		t.Error("Expected at least one component")
+	}
+}
+
+func TestExtractCompositionAPI(t *testing.T) {
+	parser := NewParser()
+
+	doc, err := goquery.NewDocumentFromReader(strings.NewReader(testReferencePageHTML))
+	if err != nil {
+		t.Fatalf("Failed to parse HTML: %v", err)
+	}
+
+	compos := parser.extractCompositionAPI(doc, "https://vuejs.org/api/")
+
+	if len(compos) == 0 {
+		t.Skip("No composition API items extracted from test HTML")
+	}
+}
+
+func TestExtractDirectives(t *testing.T) {
+	parser := NewParser()
+
+	doc, err := goquery.NewDocumentFromReader(strings.NewReader(testReferencePageHTML))
+	if err != nil {
+		t.Fatalf("Failed to parse HTML: %v", err)
+	}
+
+	directives := parser.extractDirectives(doc, "https://vuejs.org/api/")
+
+	found := false
+	for _, d := range directives {
+		if d.Name == "v-bind" {
+			found = true
+			break
+		}
+	}
+
+	if !found {
+		t.Error("Expected to find v-bind directive")
+	}
+}
+
+func TestResolveURL(t *testing.T) {
+	tests := []struct {
+		base     string
+		href     string
+		expected string
+	}{
+		{"https://vuejs.org", "/api/", "https://vuejs.org/api/"},
+		{"https://vuejs.org", "https://example.com/page", "https://example.com/page"},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.href, func(t *testing.T) {
+			got := resolveURL(tt.base, tt.href)
+			if got != tt.expected {
+				t.Errorf("resolveURL(%q, %q) = %q, want %q", tt.base, tt.href, got, tt.expected)
+			}
+		})
+	}
+}

pkg/vuedocs/types.go

@@ -0,0 +1,155 @@
+// Package vuedocs provides parsing and extraction for Vue.js documentation
+// from vuejs.org/api/.
+package vuedocs
+
+import "time"
+
+// Reference represents the Vue API reference.
+type Reference struct {
+	Version      string         `json:"version"`
+	DocURL       string         `json:"doc_url"`
+	GlobalAPI    []*API         `json:"global_api,omitempty"`
+	Composition  []*Composition `json:"composition,omitempty"`
+	Options      []*Options     `json:"options,omitempty"`
+	Directives   []*Directive   `json:"directives,omitempty"`
+	Components   []*Component   `json:"components,omitempty"`
+	SpecialAttrs []*SpecialAttr `json:"special_attrs,omitempty"`
+	FetchedAt    time.Time      `json:"fetched_at"`
+}
+
+// API represents a Vue global API.
+type API struct {
+	Name       string       `json:"name"`
+	Doc        string       `json:"doc,omitempty"`
+	Signature  string       `json:"signature,omitempty"`
+	Parameters []*Parameter `json:"parameters,omitempty"`
+	Returns    string       `json:"returns,omitempty"`
+	Examples   []*Example   `json:"examples,omitempty"`
+	Category   string       `json:"category,omitempty"`
+	DocURL     string       `json:"doc_url"`
+	Deprecated string       `json:"deprecated,omitempty"`
+}
+
+// Composition represents a Composition API function.
+type Composition struct {
+	Name       string       `json:"name"`
+	Kind       string       `json:"kind,omitempty"` // ref, reactive, computed, watch, lifecycle
+	Doc        string       `json:"doc,omitempty"`
+	Signature  string       `json:"signature,omitempty"`
+	Parameters []*Parameter `json:"parameters,omitempty"`
+	Returns    string       `json:"returns,omitempty"`
+	Examples   []*Example   `json:"examples,omitempty"`
+	DocURL     string       `json:"doc_url"`
+	Deprecated string       `json:"deprecated,omitempty"`
+}
+
+// Options represents an Options API property.
+type Options struct {
+	Name       string     `json:"name"`
+	Doc        string     `json:"doc,omitempty"`
+	Type       string     `json:"type,omitempty"`
+	Default    string     `json:"default,omitempty"`
+	Examples   []*Example `json:"examples,omitempty"`
+	Category   string     `json:"category,omitempty"` // state, lifecycle, rendering
+	DocURL     string     `json:"doc_url"`
+	Deprecated string     `json:"deprecated,omitempty"`
+}
+
+// Directive represents a Vue directive.
+type Directive struct {
+	Name       string      `json:"name"`
+	Doc        string      `json:"doc,omitempty"`
+	Usage      string      `json:"usage,omitempty"`
+	Arguments  []*Argument `json:"arguments,omitempty"`
+	Modifiers  []*Modifier `json:"modifiers,omitempty"`
+	Examples   []*Example  `json:"examples,omitempty"`
+	DocURL     string      `json:"doc_url"`
+	Deprecated string      `json:"deprecated,omitempty"`
+}
+
+// Argument represents a directive argument.
+type Argument struct {
+	Name string `json:"name"`
+	Doc  string `json:"doc,omitempty"`
+}
+
+// Modifier represents a directive modifier.
+type Modifier struct {
+	Name string `json:"name"`
+	Doc  string `json:"doc,omitempty"`
+}
+
+// Component represents a Vue built-in component.
+type Component struct {
+	Name       string     `json:"name"`
+	Doc        string     `json:"doc,omitempty"`
+	Props      []*Prop    `json:"props,omitempty"`
+	Events     []*Event   `json:"events,omitempty"`
+	Slots      []*Slot    `json:"slots,omitempty"`
+	Examples   []*Example `json:"examples,omitempty"`
+	DocURL     string     `json:"doc_url"`
+	Deprecated string     `json:"deprecated,omitempty"`
+}
+
+// Prop represents a component prop.
+type Prop struct {
+	Name       string `json:"name"`
+	Type       string `json:"type,omitempty"`
+	Doc        string `json:"doc,omitempty"`
+	Required   bool   `json:"required"`
+	Default    string `json:"default,omitempty"`
+	Deprecated string `json:"deprecated,omitempty"`
+}
+
+// Event represents a component event.
+type Event struct {
+	Name       string       `json:"name"`
+	Doc        string       `json:"doc,omitempty"`
+	Parameters []*Parameter `json:"parameters,omitempty"`
+	Deprecated string       `json:"deprecated,omitempty"`
+}
+
+// Slot represents a component slot.
+type Slot struct {
+	Name    string `json:"name"`
+	Doc     string `json:"doc,omitempty"`
+	Props   string `json:"props,omitempty"` // slot props type
+	Default bool   `json:"default"`
+}
+
+// SpecialAttr represents a special Vue attribute.
+type SpecialAttr struct {
+	Name       string     `json:"name"`
+	Doc        string     `json:"doc,omitempty"`
+	Usage      string     `json:"usage,omitempty"`
+	Examples   []*Example `json:"examples,omitempty"`
+	DocURL     string     `json:"doc_url"`
+	Deprecated string     `json:"deprecated,omitempty"`
+}
+
+// Parameter represents a function parameter.
+type Parameter struct {
+	Name     string `json:"name"`
+	Type     string `json:"type,omitempty"`
+	Doc      string `json:"doc,omitempty"`
+	Optional bool   `json:"optional"`
+	Default  string `json:"default,omitempty"`
+}
+
+// Example represents a code example.
+type Example struct {
+	Title       string `json:"title,omitempty"`
+	Code        string `json:"code"`
+	Description string `json:"description,omitempty"`
+	Language    string `json:"language,omitempty"`
+}
+
+// SearchResult represents a search result.
+type SearchResult struct {
+	Name       string `json:"name"`
+	Kind       string `json:"kind"` // api, composition, options, directive, component
+	Doc        string `json:"doc,omitempty"`
+	DocURL     string `json:"doc_url"`
+	Score      int    `json:"score"`
+	Deprecated bool   `json:"deprecated"`
+}