Devour/internal/scraper/wrapper_test.go

package scraper

import (
	"context"
	"fmt"
	"testing"
)

type flakyStubScraper struct {
	failFirst bool
	calls     int
}

func (f *flakyStubScraper) Scrape(ctx context.Context, source *Source) ([]*Document, error) {
	f.calls++
	if f.failFirst && f.calls == 1 {
		return nil, fmt.Errorf("HTTP 503")
	}
	return []*Document{
		{
			Title:   "Example ¶ deprecated",
			Content: "ok",
			URL:     source.URL,
			Type:    "test",
		},
	}, nil
}

func (f *flakyStubScraper) DetectChanges(ctx context.Context, source *Source, lastHash string) (bool, string, error) {
	return true, "hash", nil
}

func TestWrappedScraper_RetriesAndNormalizes(t *testing.T) {
	w := wrapScraper(&flakyStubScraper{failFirst: true})
	docs, err := w.Scrape(context.Background(), &Source{URL: "https://example.com"})
	if err != nil {
		t.Fatalf("expected retry to succeed, got error: %v", err)
	}
	if len(docs) != 1 {
		t.Fatalf("expected 1 document, got %d", len(docs))
	}
	if docs[0].Title != "Example" {
		t.Fatalf("expected normalized title, got %q", docs[0].Title)
	}
}