Devour/internal/scraper/registry_integration_test.go

package scraper_test

import (
	"testing"
	"time"

	basescraper "github.com/yourorg/devour/internal/scraper"
	_ "github.com/yourorg/devour/internal/scraper/external"
)

func TestLanguageScrapersAreRegistered(t *testing.T) {
	config := &basescraper.Config{
		UserAgent: "DevourTest/1.0",
		Timeout:   2 * time.Second,
	}

	supportedDocTypes := []basescraper.SourceType{
		basescraper.SourceTypeGoDocs,
		basescraper.SourceTypeRustDocs,
		basescraper.SourceTypePythonDocs,
		basescraper.SourceTypeJavaDocs,
		basescraper.SourceTypeSpringDocs,
		basescraper.SourceTypeTSDocs,
		basescraper.SourceTypeReactDocs,
		basescraper.SourceTypeVueDocs,
		basescraper.SourceTypeNuxtDocs,
		basescraper.SourceTypeMCPDocs,
		basescraper.SourceTypeDockerDocs,
		basescraper.SourceTypeCloudflareDocs,
		basescraper.SourceTypeAstroDocs,
	}

	for _, sourceType := range supportedDocTypes {
		t.Run(string(sourceType), func(t *testing.T) {
			s := basescraper.NewScraper(sourceType, config)
			if s == nil {
				t.Fatalf("NewScraper(%q) returned nil; scraper was not registered", sourceType)
			}
		})
	}
}

func TestDetectSourceType_ForSupportedDocsHosts(t *testing.T) {
	tests := []struct {
		input    string
		expected basescraper.SourceType
	}{
		{"https://pkg.go.dev/net/http", basescraper.SourceTypeGoDocs},
		{"https://docs.rs/tokio/latest/tokio/", basescraper.SourceTypeRustDocs},
		{"https://docs.python.org/3/library/asyncio.html", basescraper.SourceTypePythonDocs},
		{"https://docs.oracle.com/javase/8/docs/api/java/util/List.html", basescraper.SourceTypeJavaDocs},
		{"https://docs.spring.io/spring-boot/docs/current/reference/htmlsingle/", basescraper.SourceTypeSpringDocs},
		{"https://www.typescriptlang.org/docs/handbook/2/basic-types.html", basescraper.SourceTypeTSDocs},
		{"https://react.dev/reference/react/hooks", basescraper.SourceTypeReactDocs},
		{"https://vuejs.org/guide/introduction.html", basescraper.SourceTypeVueDocs},
		{"https://nuxt.com/docs/guide/directory-structure", basescraper.SourceTypeNuxtDocs},
		{"https://docs.docker.com/compose", basescraper.SourceTypeDockerDocs},
		{"https://hub.docker.com/mcp/server/github", basescraper.SourceTypeMCPDocs},
		{"https://developers.cloudflare.com/workers", basescraper.SourceTypeCloudflareDocs},
		{"https://docs.astro.build/en/guides/components/", basescraper.SourceTypeAstroDocs},
	}

	for _, tt := range tests {
		t.Run(tt.input, func(t *testing.T) {
			got := basescraper.DetectSourceType(tt.input)
			if got != tt.expected {
				t.Fatalf("DetectSourceType(%q) = %q, want %q", tt.input, got, tt.expected)
			}
		})
	}
}