mirror of
https://github.com/Dvorinka/Devour.git
synced 2026-06-03 20:13:03 +00:00
i dont like commits
This commit is contained in:
+51
-15
@@ -3,6 +3,7 @@ package cmd
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"path"
|
||||
@@ -79,6 +80,19 @@ type rankedDoc struct {
|
||||
searchTerm string
|
||||
}
|
||||
|
||||
type askPersistenceWarning struct {
|
||||
operation string
|
||||
cause error
|
||||
}
|
||||
|
||||
func (w *askPersistenceWarning) Error() string {
|
||||
return fmt.Sprintf("persistence warning: %s: %v", w.operation, w.cause)
|
||||
}
|
||||
|
||||
func (w *askPersistenceWarning) Unwrap() error {
|
||||
return w.cause
|
||||
}
|
||||
|
||||
func init() {
|
||||
askCmd.Flags().StringVar(&askLanguage, "lang", "", "language/framework (required)")
|
||||
askCmd.Flags().StringVarP(&askFormat, "format", "f", "json", "output format (json, text)")
|
||||
@@ -113,7 +127,7 @@ func runAsk(cmd *cobra.Command, args []string) error {
|
||||
|
||||
cfg, err := loadAppConfig()
|
||||
if err != nil {
|
||||
return err
|
||||
return fmt.Errorf("load app config for ask command: %w", err)
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Duration(askTimeoutSec)*time.Second)
|
||||
@@ -129,7 +143,10 @@ func runAsk(cmd *cobra.Command, args []string) error {
|
||||
fallbackNeeded := shouldFallbackToLive(localRanked, terms)
|
||||
|
||||
fallbackCount := 0
|
||||
fetchErrors := []string{}
|
||||
fetchErrors := []error{}
|
||||
if localErr != nil {
|
||||
fetchErrors = append(fetchErrors, fmt.Errorf("local retrieval failed: %w", localErr))
|
||||
}
|
||||
if fallbackNeeded {
|
||||
fallbackDocs, fetched, errs := fetchAskDocsFromLive(ctx, cfg, language, question, terms)
|
||||
fallbackCount = fetched
|
||||
@@ -143,7 +160,10 @@ func runAsk(cmd *cobra.Command, args []string) error {
|
||||
}
|
||||
|
||||
if len(ranked) == 0 {
|
||||
return fmt.Errorf("no docs found for %q. errors: %s", language, strings.Join(fetchErrors, "; "))
|
||||
if len(fetchErrors) == 0 {
|
||||
return fmt.Errorf("no docs found for %q", language)
|
||||
}
|
||||
return fmt.Errorf("no docs found for %q: %w", language, errors.Join(fetchErrors...))
|
||||
}
|
||||
|
||||
sort.Slice(ranked, func(i, j int) bool {
|
||||
@@ -180,6 +200,12 @@ func runAsk(cmd *cobra.Command, args []string) error {
|
||||
Confidence: computeConfidence(question, top),
|
||||
FetchedAt: time.Now(),
|
||||
}
|
||||
for _, fetchErr := range fetchErrors {
|
||||
var persistenceWarning *askPersistenceWarning
|
||||
if errors.As(fetchErr, &persistenceWarning) {
|
||||
response.Answer.Notes = append(response.Answer.Notes, persistenceWarning.Error())
|
||||
}
|
||||
}
|
||||
|
||||
switch strings.ToLower(askFormat) {
|
||||
case "text":
|
||||
@@ -290,20 +316,20 @@ func resultMatchesLanguage(result search.Result, language string) bool {
|
||||
}
|
||||
}
|
||||
|
||||
func fetchAskDocsFromLive(ctx context.Context, cfg *appconfig.Config, language, question string, terms []string) ([]rankedDoc, int, []string) {
|
||||
func fetchAskDocsFromLive(ctx context.Context, cfg *appconfig.Config, language, question string, terms []string) ([]rankedDoc, int, []error) {
|
||||
sourceType := scraper.SourceType(mapLanguageToType(language))
|
||||
if sourceType == "" {
|
||||
return nil, 0, []string{fmt.Sprintf("unsupported language: %s", language)}
|
||||
return nil, 0, []error{fmt.Errorf("unsupported language: %s", language)}
|
||||
}
|
||||
sc := toScraperConfig(cfg, 2)
|
||||
sc.MaxDepth = 1
|
||||
s := scraper.NewScraper(sourceType, sc)
|
||||
if s == nil {
|
||||
return nil, 0, []string{fmt.Sprintf("no scraper for %s (%s)", language, sourceType)}
|
||||
return nil, 0, []error{fmt.Errorf("no scraper for %s (%s)", language, sourceType)}
|
||||
}
|
||||
|
||||
var ranked []rankedDoc
|
||||
var fetchErrors []string
|
||||
var fetchErrors []error
|
||||
seenURL := make(map[string]bool)
|
||||
totalFetched := 0
|
||||
fetchedDocs := make([]*scraper.Document, 0)
|
||||
@@ -311,11 +337,11 @@ func fetchAskDocsFromLive(ctx context.Context, cfg *appconfig.Config, language,
|
||||
for _, term := range terms {
|
||||
docURLs, err := candidateDocURLs(language, term)
|
||||
if err != nil {
|
||||
fetchErrors = append(fetchErrors, fmt.Sprintf("%s: %v", term, err))
|
||||
fetchErrors = append(fetchErrors, fmt.Errorf("%s: %w", term, err))
|
||||
continue
|
||||
}
|
||||
termFetched := false
|
||||
termErrors := make([]string, 0, len(docURLs))
|
||||
termErrors := make([]error, 0, len(docURLs))
|
||||
for _, docURL := range docURLs {
|
||||
if seenURL[docURL] {
|
||||
continue
|
||||
@@ -331,11 +357,11 @@ func fetchAskDocsFromLive(ctx context.Context, cfg *appconfig.Config, language,
|
||||
|
||||
docs, err := s.Scrape(ctx, source)
|
||||
if err != nil {
|
||||
termErrors = append(termErrors, fmt.Sprintf("%s: %v", docURL, err))
|
||||
termErrors = append(termErrors, fmt.Errorf("%s: %w", docURL, err))
|
||||
continue
|
||||
}
|
||||
if len(docs) == 0 {
|
||||
termErrors = append(termErrors, fmt.Sprintf("%s: no documents extracted", docURL))
|
||||
termErrors = append(termErrors, fmt.Errorf("%s: no documents extracted", docURL))
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -353,20 +379,30 @@ func fetchAskDocsFromLive(ctx context.Context, cfg *appconfig.Config, language,
|
||||
break
|
||||
}
|
||||
if !termFetched && len(termErrors) > 0 {
|
||||
fetchErrors = append(fetchErrors, strings.Join(termErrors, " | "))
|
||||
fetchErrors = append(fetchErrors, errors.Join(termErrors...))
|
||||
}
|
||||
}
|
||||
|
||||
// Persist fallback docs for future local-first queries.
|
||||
if len(fetchedDocs) > 0 {
|
||||
_, _ = storage.SaveDocuments(fetchedDocs, storage.SaveOptions{
|
||||
if _, err := storage.SaveDocuments(fetchedDocs, storage.SaveOptions{
|
||||
Format: "json",
|
||||
OutputDir: cfg.Storage.DocsDir,
|
||||
AllowEmpty: true,
|
||||
})
|
||||
}); err != nil {
|
||||
fetchErrors = append(fetchErrors, &askPersistenceWarning{
|
||||
operation: "save fallback docs",
|
||||
cause: err,
|
||||
})
|
||||
}
|
||||
if cfg.Indexing.Enabled {
|
||||
engine := search.NewEngine(cfg)
|
||||
_, _ = engine.Rebuild(context.Background())
|
||||
if _, err := engine.Rebuild(context.Background()); err != nil {
|
||||
fetchErrors = append(fetchErrors, &askPersistenceWarning{
|
||||
operation: "rebuild index after fallback",
|
||||
cause: err,
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user