package scraper import ( "context" "crypto/sha256" "encoding/hex" "fmt" "io" "net/http" "strings" "time" "github.com/yourorg/devour/pkg/reactdocs" ) type ReactDocsScraper struct { config *Config parser *reactdocs.Parser client *http.Client } func NewReactDocsScraper(config *Config) *ReactDocsScraper { return &ReactDocsScraper{ config: config, parser: reactdocs.NewParser(), client: &http.Client{ Timeout: config.Timeout, }, } } func (s *ReactDocsScraper) Scrape(ctx context.Context, source *Source) ([]*Document, error) { var documents []*Document if source.URL == "" { return nil, fmt.Errorf("URL is required for React docs scraper") } html, err := s.fetchPage(ctx, source.URL) if err != nil { return nil, fmt.Errorf("failed to fetch page: %w", err) } ref, err := s.parser.ParseReferencePage(html, source.URL) if err != nil { return nil, fmt.Errorf("failed to parse reference: %w", err) } mainDoc := s.referenceToDocument(ref, source.Name) documents = append(documents, mainDoc) for _, hook := range ref.Hooks { doc := s.hookToDocument(hook, source.Name) documents = append(documents, doc) } for _, comp := range ref.Components { doc := s.componentToDocument(comp, source.Name) documents = append(documents, doc) } for _, api := range ref.APIs { doc := s.apiToDocument(api, source.Name) documents = append(documents, doc) } return documents, nil } func (s *ReactDocsScraper) DetectChanges(ctx context.Context, source *Source, lastHash string) (bool, string, error) { html, err := s.fetchPage(ctx, source.URL) if err != nil { return false, "", err } hash := s.generateHash(html) changed := hash != lastHash return changed, hash, nil } func (s *ReactDocsScraper) fetchPage(ctx context.Context, url string) (string, error) { req, err := http.NewRequestWithContext(ctx, "GET", url, nil) if err != nil { return "", err } req.Header.Set("User-Agent", s.config.UserAgent) resp, err := s.client.Do(req) if err != nil { return "", err } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { return "", fmt.Errorf("HTTP %d", resp.StatusCode) } body, err := io.ReadAll(resp.Body) if err != nil { return "", err } return string(body), nil } func (s *ReactDocsScraper) generateHash(content string) string { hash := sha256.Sum256([]byte(content)) return hex.EncodeToString(hash[:]) } func (s *ReactDocsScraper) referenceToDocument(ref *reactdocs.Reference, sourceName string) *Document { var content strings.Builder fmt.Fprintf(&content, "# React API Reference\n\n") fmt.Fprintf(&content, "Hooks: %d, Components: %d, APIs: %d\n", len(ref.Hooks), len(ref.Components), len(ref.APIs)) return &Document{ ID: generateDocID(ref.DocURL), Source: sourceName, Type: "react-reference", Title: "React API Reference", Content: content.String(), URL: ref.DocURL, Metadata: map[string]interface{}{"doc_type": "react-reference"}, Hash: s.generateHash(content.String()), Timestamp: time.Now(), } } func (s *ReactDocsScraper) hookToDocument(hook *reactdocs.Hook, sourceName string) *Document { var content strings.Builder fmt.Fprintf(&content, "# %s\n\n", hook.Name) if hook.Signature != "" { fmt.Fprintf(&content, "```javascript\n%s\n```\n\n", hook.Signature) } fmt.Fprintf(&content, "%s\n", hook.Doc) metadata := map[string]interface{}{ "name": hook.Name, "category": hook.Category, "doc_url": hook.DocURL, "doc_type": "react-hook", } return &Document{ ID: generateDocID(hook.DocURL), Source: sourceName, Type: "react-hook", Title: hook.Name, Content: content.String(), URL: hook.DocURL, Metadata: metadata, Hash: s.generateHash(content.String()), Timestamp: time.Now(), } } func (s *ReactDocsScraper) componentToDocument(comp *reactdocs.Component, sourceName string) *Document { var content strings.Builder fmt.Fprintf(&content, "# <%s />\n\n", comp.Name) fmt.Fprintf(&content, "%s\n", comp.Doc) if len(comp.Props) > 0 { fmt.Fprintf(&content, "\n## Props\n") for _, p := range comp.Props { fmt.Fprintf(&content, "- `%s: %s`\n", p.Name, p.Type) } } metadata := map[string]interface{}{ "name": comp.Name, "doc_url": comp.DocURL, "doc_type": "react-component", } return &Document{ ID: generateDocID(comp.DocURL), Source: sourceName, Type: "react-component", Title: comp.Name, Content: content.String(), URL: comp.DocURL, Metadata: metadata, Hash: s.generateHash(content.String()), Timestamp: time.Now(), } } func (s *ReactDocsScraper) apiToDocument(api *reactdocs.API, sourceName string) *Document { var content strings.Builder fmt.Fprintf(&content, "# %s()\n\n", api.Name) if api.Signature != "" { fmt.Fprintf(&content, "```javascript\n%s\n```\n\n", api.Signature) } fmt.Fprintf(&content, "%s\n", api.Doc) metadata := map[string]interface{}{ "name": api.Name, "doc_url": api.DocURL, "doc_type": "react-api", } return &Document{ ID: generateDocID(api.DocURL), Source: sourceName, Type: "react-api", Title: api.Name, Content: content.String(), URL: api.DocURL, Metadata: metadata, Hash: s.generateHash(content.String()), Timestamp: time.Now(), } }