update

internal/scraper/registry_integration_test.go

@@ -0,0 +1,71 @@
+package scraper_test
+
+import (
+	"testing"
+	"time"
+
+	basescraper "github.com/yourorg/devour/internal/scraper"
+	_ "github.com/yourorg/devour/internal/scraper/external"
+)
+
+func TestLanguageScrapersAreRegistered(t *testing.T) {
+	config := &basescraper.Config{
+		UserAgent: "DevourTest/1.0",
+		Timeout:   2 * time.Second,
+	}
+
+	supportedDocTypes := []basescraper.SourceType{
+		basescraper.SourceTypeGoDocs,
+		basescraper.SourceTypeRustDocs,
+		basescraper.SourceTypePythonDocs,
+		basescraper.SourceTypeJavaDocs,
+		basescraper.SourceTypeSpringDocs,
+		basescraper.SourceTypeTSDocs,
+		basescraper.SourceTypeReactDocs,
+		basescraper.SourceTypeVueDocs,
+		basescraper.SourceTypeNuxtDocs,
+		basescraper.SourceTypeMCPDocs,
+		basescraper.SourceTypeDockerDocs,
+		basescraper.SourceTypeCloudflareDocs,
+		basescraper.SourceTypeAstroDocs,
+	}
+
+	for _, sourceType := range supportedDocTypes {
+		t.Run(string(sourceType), func(t *testing.T) {
+			s := basescraper.NewScraper(sourceType, config)
+			if s == nil {
+				t.Fatalf("NewScraper(%q) returned nil; scraper was not registered", sourceType)
+			}
+		})
+	}
+}
+
+func TestDetectSourceType_ForSupportedDocsHosts(t *testing.T) {
+	tests := []struct {
+		input    string
+		expected basescraper.SourceType
+	}{
+		{"https://pkg.go.dev/net/http", basescraper.SourceTypeGoDocs},
+		{"https://docs.rs/tokio/latest/tokio/", basescraper.SourceTypeRustDocs},
+		{"https://docs.python.org/3/library/asyncio.html", basescraper.SourceTypePythonDocs},
+		{"https://docs.oracle.com/javase/8/docs/api/java/util/List.html", basescraper.SourceTypeJavaDocs},
+		{"https://docs.spring.io/spring-boot/docs/current/reference/htmlsingle/", basescraper.SourceTypeSpringDocs},
+		{"https://www.typescriptlang.org/docs/handbook/2/basic-types.html", basescraper.SourceTypeTSDocs},
+		{"https://react.dev/reference/react/hooks", basescraper.SourceTypeReactDocs},
+		{"https://vuejs.org/guide/introduction.html", basescraper.SourceTypeVueDocs},
+		{"https://nuxt.com/docs/guide/directory-structure", basescraper.SourceTypeNuxtDocs},
+		{"https://docs.docker.com/compose", basescraper.SourceTypeDockerDocs},
+		{"https://hub.docker.com/mcp/server/github", basescraper.SourceTypeMCPDocs},
+		{"https://developers.cloudflare.com/workers", basescraper.SourceTypeCloudflareDocs},
+		{"https://docs.astro.build/en/guides/components/", basescraper.SourceTypeAstroDocs},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.input, func(t *testing.T) {
+			got := basescraper.DetectSourceType(tt.input)
+			if got != tt.expected {
+				t.Fatalf("DetectSourceType(%q) = %q, want %q", tt.input, got, tt.expected)
+			}
+		})
+	}
+}