mirror of
https://github.com/Dvorinka/Devour.git
synced 2026-06-04 12:33:04 +00:00
Tomas Dvorak
57 lines
1.5 KiB
Go
57 lines
1.5 KiB
Go
package cmd
|
|
|
|
import (
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"testing"
|
|
|
|
appconfig "github.com/yourorg/devour/internal/config"
|
|
)
|
|
|
|
func TestScrapeFromConfig(t *testing.T) {
|
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
w.Header().Set("Content-Type", "text/html")
|
|
_, _ = w.Write([]byte("<html><head><title>Docs</title></head><body><main>" + strings.Repeat("docs content ", 30) + "</main></body></html>"))
|
|
}))
|
|
defer srv.Close()
|
|
|
|
tmp := t.TempDir()
|
|
cfg := appconfig.Default()
|
|
cfg.Storage.DocsDir = filepath.Join(tmp, "docs")
|
|
cfg.Storage.IndexDir = filepath.Join(tmp, "index")
|
|
cfg.Storage.MetadataDir = filepath.Join(tmp, "metadata")
|
|
cfg.Storage.CacheDir = filepath.Join(tmp, "cache")
|
|
if err := cfg.EnsureStorageDirs(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
sourcesPath := filepath.Join(tmp, "sources.yaml")
|
|
yaml := "- name: demo\n type: url\n url: " + srv.URL + "\n"
|
|
if err := os.WriteFile(sourcesPath, []byte(yaml), 0o644); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
oldFormat, oldOutput, oldAllow := scrapeFormat, scrapeOutput, scrapeAllowEmpty
|
|
scrapeFormat = "json"
|
|
scrapeOutput = cfg.Storage.DocsDir
|
|
scrapeAllowEmpty = false
|
|
defer func() {
|
|
scrapeFormat, scrapeOutput, scrapeAllowEmpty = oldFormat, oldOutput, oldAllow
|
|
}()
|
|
|
|
if err := scrapeFromConfig(nil, cfg, sourcesPath); err != nil {
|
|
t.Fatalf("scrapeFromConfig failed: %v", err)
|
|
}
|
|
|
|
entries, err := os.ReadDir(cfg.Storage.DocsDir)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(entries) == 0 {
|
|
t.Fatal("expected scraped files")
|
|
}
|
|
}
|