Devour/internal/scraper/local_test.go

package scraper

import (
	"context"
	"os"
	"path/filepath"
	"strings"
	"testing"
	"time"
)

func TestLocalScraperScrapeDirectory(t *testing.T) {
	tmp := t.TempDir()
	if err := os.WriteFile(filepath.Join(tmp, "README.md"), []byte("# Demo\n\nhello docs"), 0o644); err != nil {
		t.Fatal(err)
	}
	if err := os.WriteFile(filepath.Join(tmp, "notes.txt"), []byte("notes"), 0o644); err != nil {
		t.Fatal(err)
	}
	if err := os.WriteFile(filepath.Join(tmp, "bin.bin"), []byte{0x00, 0x01}, 0o644); err != nil {
		t.Fatal(err)
	}

	s := NewLocalScraper(&Config{Timeout: 2 * time.Second, UserAgent: "DevourTest"})
	docs, err := s.Scrape(context.Background(), &Source{Name: "local", Type: SourceTypeLocal, Path: tmp})
	if err != nil {
		t.Fatalf("unexpected error: %v", err)
	}
	if len(docs) < 2 {
		t.Fatalf("expected at least 2 docs, got %d", len(docs))
	}
}

func TestLocalScraperDetectChanges(t *testing.T) {
	tmp := t.TempDir()
	file := filepath.Join(tmp, "README.md")
	if err := os.WriteFile(file, []byte("v1"), 0o644); err != nil {
		t.Fatal(err)
	}

	s := NewLocalScraper(&Config{Timeout: 2 * time.Second, UserAgent: "DevourTest"})
	src := &Source{Name: "local", Type: SourceTypeLocal, Path: tmp}

	changed, hash1, err := s.DetectChanges(context.Background(), src, "")
	if err != nil {
		t.Fatal(err)
	}
	if !changed || hash1 == "" {
		t.Fatalf("expected first detect to change with non-empty hash, changed=%v hash=%q", changed, hash1)
	}

	time.Sleep(5 * time.Millisecond)
	if err := os.WriteFile(file, []byte("v2"), 0o644); err != nil {
		t.Fatal(err)
	}

	changed, hash2, err := s.DetectChanges(context.Background(), src, hash1)
	if err != nil {
		t.Fatal(err)
	}
	if !changed {
		t.Fatal("expected change after file update")
	}
	if hash1 == hash2 {
		t.Fatal("expected hash to change")
	}
}

func TestLocalScraper_StripsFrontmatterAndMDXPreamble(t *testing.T) {
	tmp := t.TempDir()
	path := filepath.Join(tmp, "doc.mdx")
	content := `---
title: My Doc
slug: /my-doc
---

import { Component } from "x"
export const meta = {}

# Heading

Actual documentation body.
`
	if err := os.WriteFile(path, []byte(content), 0o644); err != nil {
		t.Fatal(err)
	}

	s := NewLocalScraper(&Config{Timeout: 2 * time.Second, UserAgent: "DevourTest"})
	doc, err := s.fileToDocument(path, &Source{Name: "local", Type: SourceTypeLocal, Path: tmp})
	if err != nil {
		t.Fatalf("unexpected error: %v", err)
	}
	if strings.Contains(doc.Content, "slug: /my-doc") {
		t.Fatalf("expected frontmatter to be stripped, got: %q", doc.Content)
	}
	if strings.Contains(doc.Content, "import { Component }") {
		t.Fatalf("expected MDX preamble to be stripped, got: %q", doc.Content)
	}
	if !strings.Contains(doc.Content, "Actual documentation body.") {
		t.Fatalf("expected markdown body in content, got: %q", doc.Content)
	}
}