update

internal/scraper/normalize_test.go

@@ -0,0 +1,33 @@
+package scraper
+
+import "testing"
+
+func TestNormalizeDocument_TitleCleanup(t *testing.T) {
+	doc := &Document{
+		Title:   "http.type CloseNotifier ¶ deprecated added in go1.1",
+		Content: "line 1  \n\n\nline 2",
+		URL:     "https://pkg.go.dev/net/http#CloseNotifier",
+	}
+
+	NormalizeDocument(doc)
+
+	if doc.Title != "http.type CloseNotifier" {
+		t.Fatalf("unexpected normalized title: %q", doc.Title)
+	}
+	if doc.Content != "line 1\n\nline 2" {
+		t.Fatalf("unexpected normalized content: %q", doc.Content)
+	}
+}
+
+func TestNormalizeDocument_InferTitle(t *testing.T) {
+	doc := &Document{
+		Title: "",
+		URL:   "https://kotlinlang.org/docs/regex.html",
+	}
+
+	NormalizeDocument(doc)
+
+	if doc.Title != "regex" {
+		t.Fatalf("expected inferred title 'regex', got %q", doc.Title)
+	}
+}