update

2026-06-03 20:13:03 +00:00 · 2026-02-24 10:33:59 +01:00
parent 409acd2e08
commit 898a3c303f
1374 changed files with 290409 additions and 29187 deletions
@@ -1,11 +1,11 @@
 package mcpdocs

 import (
-	"net/url"
 	"strings"
 	"time"

 	"github.com/PuerkitoBio/goquery"
+	"github.com/yourorg/devour/pkg/parserutil"
 )

 type Parser struct {
@@ -194,19 +194,5 @@ func (p *Parser) extractPrompts(doc *goquery.Document, docURL string) []*Prompt
 }

 func resolveURL(base string, href string) string {
-	if strings.HasPrefix(href, "http") {
-		return href
-	}
-
-	baseURL, err := url.Parse(base)
-	if err != nil {
-		return href
-	}
-
-	hrefURL, err := url.Parse(href)
-	if err != nil {
-		return href
-	}
-
-	return baseURL.ResolveReference(hrefURL).String()
+	return parserutil.ResolveURL(base, href)
 }
@@ -0,0 +1,102 @@
+package mcpdocs
+
+import (
+	"strings"
+	"testing"
+)
+
+const testMCPServerPageHTML = `
+<!DOCTYPE html>
+<html>
+<head>
+  <title>GitHub MCP Server | Docker Hub</title>
+  <meta name="description" content="MCP server for GitHub automation.">
+  <meta property="og:image" content="https://hub.docker.com/image.png">
+</head>
+<body>
+  <span class="category">developer-tools</span>
+  <main>
+    <h1>GitHub MCP Server</h1>
+
+    <h2>Tools</h2>
+    <ul>
+      <li><code>search_repos</code><p>Search repositories.</p></li>
+      <li><code>open_pr</code><p>Create pull requests.</p></li>
+    </ul>
+
+    <h2>Resources</h2>
+    <ul>
+      <li><code>repo_readme</code><p>Repository readme content.</p></li>
+    </ul>
+
+    <h2>Prompts</h2>
+    <ul>
+      <li><code>summarize_pr</code><p>Summarize pull requests.</p></li>
+    </ul>
+  </main>
+</body>
+</html>
+`
+
+func TestParseServerPage(t *testing.T) {
+	parser := NewParser()
+
+	server, err := parser.ParseServerPage(testMCPServerPageHTML, "https://hub.docker.com/mcp/server/github")
+	if err != nil {
+		t.Fatalf("ParseServerPage failed: %v", err)
+	}
+
+	if server.Name != "GitHub MCP Server" {
+		t.Fatalf("unexpected server name: %q", server.Name)
+	}
+	if server.Description == "" {
+		t.Fatal("expected non-empty description")
+	}
+	if strings.TrimSpace(server.Category) != "developer-tools" {
+		t.Fatalf("unexpected category: %q", server.Category)
+	}
+	if len(server.Tools) != 2 {
+		t.Fatalf("expected 2 tools, got %d", len(server.Tools))
+	}
+	if len(server.Resources) != 1 {
+		t.Fatalf("expected 1 resource, got %d", len(server.Resources))
+	}
+	if len(server.Prompts) != 1 {
+		t.Fatalf("expected 1 prompt, got %d", len(server.Prompts))
+	}
+}
+
+func TestParseHubPage(t *testing.T) {
+	parser := NewParser()
+
+	html := `
+<div>
+  <a href="/mcp/server/github">
+    <h3>GitHub MCP Server</h3>
+    <p>MCP server for GitHub automation.</p>
+  </a>
+  <a href="/mcp/server/slack">
+    <h3>Slack MCP Server</h3>
+    <p>MCP server for Slack workflows.</p>
+  </a>
+</div>`
+
+	servers, err := parser.ParseHubPage(html)
+	if err != nil {
+		t.Fatalf("ParseHubPage failed: %v", err)
+	}
+
+	if len(servers) != 2 {
+		t.Fatalf("expected 2 servers, got %d", len(servers))
+	}
+	if servers[0].DocURL != "https://hub.docker.com/mcp/server/github" {
+		t.Fatalf("unexpected resolved URL: %q", servers[0].DocURL)
+	}
+}
+
+func TestResolveURL(t *testing.T) {
+	got := resolveURL("https://hub.docker.com", "/mcp/server/github")
+	if got != "https://hub.docker.com/mcp/server/github" {
+		t.Fatalf("resolveURL returned %q", got)
+	}
+}