updage

internal/scraper/external/reactdocs.go

@@ -0,0 +1,214 @@
+package scraper
+
+import (
+	"context"
+	"crypto/sha256"
+	"encoding/hex"
+	"fmt"
+	"io"
+	"net/http"
+	"strings"
+	"time"
+
+	"github.com/yourorg/devour/pkg/reactdocs"
+)
+
+type ReactDocsScraper struct {
+	config *Config
+	parser *reactdocs.Parser
+	client *http.Client
+}
+
+func NewReactDocsScraper(config *Config) *ReactDocsScraper {
+	return &ReactDocsScraper{
+		config: config,
+		parser: reactdocs.NewParser(),
+		client: &http.Client{
+			Timeout: config.Timeout,
+		},
+	}
+}
+
+func (s *ReactDocsScraper) Scrape(ctx context.Context, source *Source) ([]*Document, error) {
+	var documents []*Document
+
+	if source.URL == "" {
+		return nil, fmt.Errorf("URL is required for React docs scraper")
+	}
+
+	html, err := s.fetchPage(ctx, source.URL)
+	if err != nil {
+		return nil, fmt.Errorf("failed to fetch page: %w", err)
+	}
+
+	ref, err := s.parser.ParseReferencePage(html, source.URL)
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse reference: %w", err)
+	}
+
+	mainDoc := s.referenceToDocument(ref, source.Name)
+	documents = append(documents, mainDoc)
+
+	for _, hook := range ref.Hooks {
+		doc := s.hookToDocument(hook, source.Name)
+		documents = append(documents, doc)
+	}
+
+	for _, comp := range ref.Components {
+		doc := s.componentToDocument(comp, source.Name)
+		documents = append(documents, doc)
+	}
+
+	for _, api := range ref.APIs {
+		doc := s.apiToDocument(api, source.Name)
+		documents = append(documents, doc)
+	}
+
+	return documents, nil
+}
+
+func (s *ReactDocsScraper) DetectChanges(ctx context.Context, source *Source, lastHash string) (bool, string, error) {
+	html, err := s.fetchPage(ctx, source.URL)
+	if err != nil {
+		return false, "", err
+	}
+
+	hash := s.generateHash(html)
+	changed := hash != lastHash
+
+	return changed, hash, nil
+}
+
+func (s *ReactDocsScraper) fetchPage(ctx context.Context, url string) (string, error) {
+	req, err := http.NewRequestWithContext(ctx, "GET", url, nil)
+	if err != nil {
+		return "", err
+	}
+
+	req.Header.Set("User-Agent", s.config.UserAgent)
+
+	resp, err := s.client.Do(req)
+	if err != nil {
+		return "", err
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		return "", fmt.Errorf("HTTP %d", resp.StatusCode)
+	}
+
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return "", err
+	}
+	return string(body), nil
+}
+
+func (s *ReactDocsScraper) generateHash(content string) string {
+	hash := sha256.Sum256([]byte(content))
+	return hex.EncodeToString(hash[:])
+}
+
+func (s *ReactDocsScraper) referenceToDocument(ref *reactdocs.Reference, sourceName string) *Document {
+	var content strings.Builder
+	fmt.Fprintf(&content, "# React API Reference\n\n")
+	fmt.Fprintf(&content, "Hooks: %d, Components: %d, APIs: %d\n", len(ref.Hooks), len(ref.Components), len(ref.APIs))
+
+	return &Document{
+		ID:        generateDocID(ref.DocURL),
+		Source:    sourceName,
+		Type:      "react-reference",
+		Title:     "React API Reference",
+		Content:   content.String(),
+		URL:       ref.DocURL,
+		Metadata:  map[string]interface{}{"doc_type": "react-reference"},
+		Hash:      s.generateHash(content.String()),
+		Timestamp: time.Now(),
+	}
+}
+
+func (s *ReactDocsScraper) hookToDocument(hook *reactdocs.Hook, sourceName string) *Document {
+	var content strings.Builder
+	fmt.Fprintf(&content, "# %s\n\n", hook.Name)
+	if hook.Signature != "" {
+		fmt.Fprintf(&content, "```javascript\n%s\n```\n\n", hook.Signature)
+	}
+	fmt.Fprintf(&content, "%s\n", hook.Doc)
+
+	metadata := map[string]interface{}{
+		"name":     hook.Name,
+		"category": hook.Category,
+		"doc_url":  hook.DocURL,
+		"doc_type": "react-hook",
+	}
+
+	return &Document{
+		ID:        generateDocID(hook.DocURL),
+		Source:    sourceName,
+		Type:      "react-hook",
+		Title:     hook.Name,
+		Content:   content.String(),
+		URL:       hook.DocURL,
+		Metadata:  metadata,
+		Hash:      s.generateHash(content.String()),
+		Timestamp: time.Now(),
+	}
+}
+
+func (s *ReactDocsScraper) componentToDocument(comp *reactdocs.Component, sourceName string) *Document {
+	var content strings.Builder
+	fmt.Fprintf(&content, "# <%s />\n\n", comp.Name)
+	fmt.Fprintf(&content, "%s\n", comp.Doc)
+
+	if len(comp.Props) > 0 {
+		fmt.Fprintf(&content, "\n## Props\n")
+		for _, p := range comp.Props {
+			fmt.Fprintf(&content, "- `%s: %s`\n", p.Name, p.Type)
+		}
+	}
+
+	metadata := map[string]interface{}{
+		"name":     comp.Name,
+		"doc_url":  comp.DocURL,
+		"doc_type": "react-component",
+	}
+
+	return &Document{
+		ID:        generateDocID(comp.DocURL),
+		Source:    sourceName,
+		Type:      "react-component",
+		Title:     comp.Name,
+		Content:   content.String(),
+		URL:       comp.DocURL,
+		Metadata:  metadata,
+		Hash:      s.generateHash(content.String()),
+		Timestamp: time.Now(),
+	}
+}
+
+func (s *ReactDocsScraper) apiToDocument(api *reactdocs.API, sourceName string) *Document {
+	var content strings.Builder
+	fmt.Fprintf(&content, "# %s()\n\n", api.Name)
+	if api.Signature != "" {
+		fmt.Fprintf(&content, "```javascript\n%s\n```\n\n", api.Signature)
+	}
+	fmt.Fprintf(&content, "%s\n", api.Doc)
+
+	metadata := map[string]interface{}{
+		"name":     api.Name,
+		"doc_url":  api.DocURL,
+		"doc_type": "react-api",
+	}
+
+	return &Document{
+		ID:        generateDocID(api.DocURL),
+		Source:    sourceName,
+		Type:      "react-api",
+		Title:     api.Name,
+		Content:   content.String(),
+		URL:       api.DocURL,
+		Metadata:  metadata,
+		Hash:      s.generateHash(content.String()),
+		Timestamp: time.Now(),
+	}
+}