package cmd
import (
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strings"
"testing"
appconfig "github.com/yourorg/devour/internal/config"
)
func TestScrapeFromConfig(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "text/html")
_, _ = w.Write([]byte("Docs" + strings.Repeat("docs content ", 30) + ""))
}))
defer srv.Close()
tmp := t.TempDir()
cfg := appconfig.Default()
cfg.Storage.DocsDir = filepath.Join(tmp, "docs")
cfg.Storage.IndexDir = filepath.Join(tmp, "index")
cfg.Storage.MetadataDir = filepath.Join(tmp, "metadata")
cfg.Storage.CacheDir = filepath.Join(tmp, "cache")
if err := cfg.EnsureStorageDirs(); err != nil {
t.Fatal(err)
}
sourcesPath := filepath.Join(tmp, "sources.yaml")
yaml := "- name: demo\n type: url\n url: " + srv.URL + "\n"
if err := os.WriteFile(sourcesPath, []byte(yaml), 0o644); err != nil {
t.Fatal(err)
}
oldFormat, oldOutput, oldAllow := scrapeFormat, scrapeOutput, scrapeAllowEmpty
scrapeFormat = "json"
scrapeOutput = cfg.Storage.DocsDir
scrapeAllowEmpty = false
defer func() {
scrapeFormat, scrapeOutput, scrapeAllowEmpty = oldFormat, oldOutput, oldAllow
}()
if err := scrapeFromConfig(nil, cfg, sourcesPath); err != nil {
t.Fatalf("scrapeFromConfig failed: %v", err)
}
entries, err := os.ReadDir(cfg.Storage.DocsDir)
if err != nil {
t.Fatal(err)
}
if len(entries) == 0 {
t.Fatal("expected scraped files")
}
}