mirror of
https://github.com/Dvorinka/Devour.git
synced 2026-06-03 20:13:03 +00:00
updage
This commit is contained in:
Tomas Dvorak
Vendored
+261
@@ -0,0 +1,261 @@
|
||||
package scraper
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/yourorg/devour/pkg/tsdocs"
|
||||
)
|
||||
|
||||
type TSDocsScraper struct {
|
||||
config *Config
|
||||
parser *tsdocs.Parser
|
||||
client *http.Client
|
||||
}
|
||||
|
||||
func NewTSDocsScraper(config *Config) *TSDocsScraper {
|
||||
return &TSDocsScraper{
|
||||
config: config,
|
||||
parser: tsdocs.NewParser(),
|
||||
client: &http.Client{
|
||||
Timeout: config.Timeout,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func (s *TSDocsScraper) Scrape(ctx context.Context, source *Source) ([]*Document, error) {
|
||||
var documents []*Document
|
||||
|
||||
if source.URL == "" {
|
||||
return nil, fmt.Errorf("URL is required for TypeScript docs scraper")
|
||||
}
|
||||
|
||||
html, err := s.fetchPage(ctx, source.URL)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to fetch page: %w", err)
|
||||
}
|
||||
|
||||
module, err := s.parser.ParseModulePage(html, source.URL)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to parse module: %w", err)
|
||||
}
|
||||
|
||||
mainDoc := s.moduleToDocument(module, source.Name)
|
||||
documents = append(documents, mainDoc)
|
||||
|
||||
for _, iface := range module.Interfaces {
|
||||
doc := s.interfaceToDocument(iface, module, source.Name)
|
||||
documents = append(documents, doc)
|
||||
}
|
||||
|
||||
for _, fn := range module.Functions {
|
||||
doc := s.functionToDocument(fn, module, source.Name)
|
||||
documents = append(documents, doc)
|
||||
}
|
||||
|
||||
for _, class := range module.Classes {
|
||||
doc := s.classToDocument(class, module, source.Name)
|
||||
documents = append(documents, doc)
|
||||
}
|
||||
|
||||
for _, ta := range module.Types {
|
||||
doc := s.typeAliasToDocument(ta, module, source.Name)
|
||||
documents = append(documents, doc)
|
||||
}
|
||||
|
||||
return documents, nil
|
||||
}
|
||||
|
||||
func (s *TSDocsScraper) DetectChanges(ctx context.Context, source *Source, lastHash string) (bool, string, error) {
|
||||
html, err := s.fetchPage(ctx, source.URL)
|
||||
if err != nil {
|
||||
return false, "", err
|
||||
}
|
||||
|
||||
hash := s.generateHash(html)
|
||||
changed := hash != lastHash
|
||||
|
||||
return changed, hash, nil
|
||||
}
|
||||
|
||||
func (s *TSDocsScraper) fetchPage(ctx context.Context, url string) (string, error) {
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", url, nil)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
req.Header.Set("User-Agent", s.config.UserAgent)
|
||||
|
||||
resp, err := s.client.Do(req)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return "", fmt.Errorf("HTTP %d", resp.StatusCode)
|
||||
}
|
||||
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return string(body), nil
|
||||
}
|
||||
|
||||
func (s *TSDocsScraper) generateHash(content string) string {
|
||||
hash := sha256.Sum256([]byte(content))
|
||||
return hex.EncodeToString(hash[:])
|
||||
}
|
||||
|
||||
func (s *TSDocsScraper) moduleToDocument(module *tsdocs.Module, sourceName string) *Document {
|
||||
var content strings.Builder
|
||||
fmt.Fprintf(&content, "# %s\n\n", module.Name)
|
||||
fmt.Fprintf(&content, "%s\n", module.Doc)
|
||||
|
||||
metadata := map[string]interface{}{
|
||||
"module": module.Name,
|
||||
"version": module.Version,
|
||||
"doc_url": module.DocURL,
|
||||
"doc_type": "ts-module",
|
||||
}
|
||||
|
||||
return &Document{
|
||||
ID: generateDocID(module.DocURL),
|
||||
Source: sourceName,
|
||||
Type: "ts-module",
|
||||
Title: module.Name,
|
||||
Content: content.String(),
|
||||
URL: module.DocURL,
|
||||
Metadata: metadata,
|
||||
Hash: s.generateHash(content.String()),
|
||||
Timestamp: time.Now(),
|
||||
}
|
||||
}
|
||||
|
||||
func (s *TSDocsScraper) interfaceToDocument(iface *tsdocs.Interface, module *tsdocs.Module, sourceName string) *Document {
|
||||
var content strings.Builder
|
||||
fmt.Fprintf(&content, "# %s (interface)\n\n", iface.Name)
|
||||
if iface.Doc != "" {
|
||||
fmt.Fprintf(&content, "%s\n\n", iface.Doc)
|
||||
}
|
||||
if len(iface.Properties) > 0 {
|
||||
fmt.Fprintf(&content, "## Properties\n")
|
||||
for _, p := range iface.Properties {
|
||||
fmt.Fprintf(&content, "- `%s: %s`\n", p.Name, p.Type)
|
||||
}
|
||||
}
|
||||
|
||||
metadata := map[string]interface{}{
|
||||
"module": module.Name,
|
||||
"name": iface.Name,
|
||||
"doc_url": iface.DocURL,
|
||||
}
|
||||
|
||||
return &Document{
|
||||
ID: generateDocID(iface.DocURL),
|
||||
Source: sourceName,
|
||||
Type: "ts-interface",
|
||||
Title: iface.Name,
|
||||
Content: content.String(),
|
||||
URL: iface.DocURL,
|
||||
Metadata: metadata,
|
||||
Hash: s.generateHash(content.String()),
|
||||
Timestamp: time.Now(),
|
||||
}
|
||||
}
|
||||
|
||||
func (s *TSDocsScraper) functionToDocument(fn *tsdocs.Function, module *tsdocs.Module, sourceName string) *Document {
|
||||
var content strings.Builder
|
||||
fmt.Fprintf(&content, "# %s()\n\n", fn.Name)
|
||||
if fn.Signature != "" {
|
||||
fmt.Fprintf(&content, "```typescript\n%s\n```\n\n", fn.Signature)
|
||||
}
|
||||
if fn.Doc != "" {
|
||||
fmt.Fprintf(&content, "%s\n", fn.Doc)
|
||||
}
|
||||
|
||||
metadata := map[string]interface{}{
|
||||
"module": module.Name,
|
||||
"name": fn.Name,
|
||||
"return_type": fn.ReturnType,
|
||||
"doc_url": fn.DocURL,
|
||||
}
|
||||
|
||||
return &Document{
|
||||
ID: generateDocID(fn.DocURL),
|
||||
Source: sourceName,
|
||||
Type: "ts-function",
|
||||
Title: fn.Name,
|
||||
Content: content.String(),
|
||||
URL: fn.DocURL,
|
||||
Metadata: metadata,
|
||||
Hash: s.generateHash(content.String()),
|
||||
Timestamp: time.Now(),
|
||||
}
|
||||
}
|
||||
|
||||
func (s *TSDocsScraper) classToDocument(class *tsdocs.Class, module *tsdocs.Module, sourceName string) *Document {
|
||||
var content strings.Builder
|
||||
fmt.Fprintf(&content, "# %s (class)\n\n", class.Name)
|
||||
if class.Doc != "" {
|
||||
fmt.Fprintf(&content, "%s\n\n", class.Doc)
|
||||
}
|
||||
if len(class.Methods) > 0 {
|
||||
fmt.Fprintf(&content, "## Methods\n")
|
||||
for _, m := range class.Methods {
|
||||
fmt.Fprintf(&content, "- `%s()`\n", m.Name)
|
||||
}
|
||||
}
|
||||
|
||||
metadata := map[string]interface{}{
|
||||
"module": module.Name,
|
||||
"name": class.Name,
|
||||
"doc_url": class.DocURL,
|
||||
}
|
||||
|
||||
return &Document{
|
||||
ID: generateDocID(class.DocURL),
|
||||
Source: sourceName,
|
||||
Type: "ts-class",
|
||||
Title: class.Name,
|
||||
Content: content.String(),
|
||||
URL: class.DocURL,
|
||||
Metadata: metadata,
|
||||
Hash: s.generateHash(content.String()),
|
||||
Timestamp: time.Now(),
|
||||
}
|
||||
}
|
||||
|
||||
func (s *TSDocsScraper) typeAliasToDocument(ta *tsdocs.TypeAlias, module *tsdocs.Module, sourceName string) *Document {
|
||||
var content strings.Builder
|
||||
fmt.Fprintf(&content, "# %s (type)\n\n", ta.Name)
|
||||
fmt.Fprintf(&content, "```typescript\ntype %s = %s\n```\n\n", ta.Name, ta.Type)
|
||||
if ta.Doc != "" {
|
||||
fmt.Fprintf(&content, "%s\n", ta.Doc)
|
||||
}
|
||||
|
||||
metadata := map[string]interface{}{
|
||||
"module": module.Name,
|
||||
"name": ta.Name,
|
||||
"doc_url": ta.DocURL,
|
||||
}
|
||||
|
||||
return &Document{
|
||||
ID: generateDocID(ta.DocURL),
|
||||
Source: sourceName,
|
||||
Type: "ts-type",
|
||||
Title: ta.Name,
|
||||
Content: content.String(),
|
||||
URL: ta.DocURL,
|
||||
Metadata: metadata,
|
||||
Hash: s.generateHash(content.String()),
|
||||
Timestamp: time.Now(),
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user