TDvorak/Devour
1
0
Fork 0
mirror of https://github.com/Dvorinka/Devour.git synced 2026-06-04 12:33:04 +00:00
Code Issues Packages Projects Releases Wiki Activity

i dont like commits

Browse Source
This commit is contained in:
Tomas Dvorak
2026-02-24 12:10:13 +01:00
parent 898a3c303f
commit 1d72a1cc01
109 changed files with 43586 additions and 8484 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/scraper/localsearch.go
+4 -3
View File
@@ -5,6 +5,7 @@ import (
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
@@ -81,7 +82,7 @@ func (s *LocalSearchScraper) Scrape(ctx context.Context, source *Source) ([]*Doc
docs := make([]*Document, 0, limit)
seen := make(map[string]bool)
var scrapeErrors []string
var scrapeErrors []error
for i, result := range results {
if ctx.Err() != nil {
@@ -109,7 +110,7 @@ func (s *LocalSearchScraper) Scrape(ctx context.Context, source *Source) ([]*Doc
})
if err != nil {
if len(scrapeErrors) < 20 {
scrapeErrors = append(scrapeErrors, fmt.Sprintf("%s: %v", resultURL, err))
scrapeErrors = append(scrapeErrors, fmt.Errorf("%s: %w", resultURL, err))
}
continue
}
@@ -140,7 +141,7 @@ func (s *LocalSearchScraper) Scrape(ctx context.Context, source *Source) ([]*Doc
if len(docs) == 0 {
if len(scrapeErrors) > 0 {
return nil, fmt.Errorf("local search returned results but page scraping failed: %s", strings.Join(scrapeErrors, "; "))
return nil, fmt.Errorf("local search returned results but page scraping failed: %w", errors.Join(scrapeErrors...))
}
return nil, fmt.Errorf("local search yielded no usable results for query %q", query)
}