update

2026-06-04 04:23:02 +00:00 · 2026-02-24 10:33:59 +01:00
parent 409acd2e08
commit 898a3c303f
1374 changed files with 290409 additions and 29187 deletions
@@ -0,0 +1,56 @@
+package cmd
+
+import (
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	appconfig "github.com/yourorg/devour/internal/config"
+)
+
+func TestScrapeFromConfig(t *testing.T) {
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "text/html")
+		_, _ = w.Write([]byte("<html><head><title>Docs</title></head><body><main>" + strings.Repeat("docs content ", 30) + "</main></body></html>"))
+	}))
+	defer srv.Close()
+
+	tmp := t.TempDir()
+	cfg := appconfig.Default()
+	cfg.Storage.DocsDir = filepath.Join(tmp, "docs")
+	cfg.Storage.IndexDir = filepath.Join(tmp, "index")
+	cfg.Storage.MetadataDir = filepath.Join(tmp, "metadata")
+	cfg.Storage.CacheDir = filepath.Join(tmp, "cache")
+	if err := cfg.EnsureStorageDirs(); err != nil {
+		t.Fatal(err)
+	}
+
+	sourcesPath := filepath.Join(tmp, "sources.yaml")
+	yaml := "- name: demo\n  type: url\n  url: " + srv.URL + "\n"
+	if err := os.WriteFile(sourcesPath, []byte(yaml), 0o644); err != nil {
+		t.Fatal(err)
+	}
+
+	oldFormat, oldOutput, oldAllow := scrapeFormat, scrapeOutput, scrapeAllowEmpty
+	scrapeFormat = "json"
+	scrapeOutput = cfg.Storage.DocsDir
+	scrapeAllowEmpty = false
+	defer func() {
+		scrapeFormat, scrapeOutput, scrapeAllowEmpty = oldFormat, oldOutput, oldAllow
+	}()
+
+	if err := scrapeFromConfig(nil, cfg, sourcesPath); err != nil {
+		t.Fatalf("scrapeFromConfig failed: %v", err)
+	}
+
+	entries, err := os.ReadDir(cfg.Storage.DocsDir)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(entries) == 0 {
+		t.Fatal("expected scraped files")
+	}
+}