feat(frontend): enhance API credentials system and build configuration

Add real API support in demo mode with credential checking, implement build-time version injection from package.json, and refactor update checking with 24-hour caching. Migrate landing page from Vue to Astro with comprehensive UI components including Hero, Features, Benefits, and Tech Stack sections. Update CI/CD workflow with expanded cache paths and security scanner version pinned.
2026-06-03 20:12:58 +00:00 · 2026-02-10 16:25:57 +01:00
parent d27cf14110
commit b083dac3f0
95 changed files with 17610 additions and 2692 deletions
@@ -0,0 +1,370 @@
+package services
+
+import (
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"regexp"
+	"strings"
+	"time"
+)
+
+// FaviconFetcher handles comprehensive favicon detection and fetching
+type FaviconFetcher struct {
+	client *http.Client
+}
+
+// NewFaviconFetcher creates a new favicon fetcher instance
+func NewFaviconFetcher() *FaviconFetcher {
+	return &FaviconFetcher{
+		client: &http.Client{
+			Timeout: 10 * time.Second,
+		},
+	}
+}
+
+// FetchFavicon fetches the best available favicon for a given URL
+func (ff *FaviconFetcher) FetchFavicon(targetURL string) (string, error) {
+	parsedURL, err := url.Parse(targetURL)
+	if err != nil {
+		return "", fmt.Errorf("invalid URL: %w", err)
+	}
+
+	// Try to extract favicon from HTML head first
+	faviconURL, err := ff.extractFromHTML(targetURL, parsedURL)
+	if err == nil && faviconURL != "" {
+		// Verify the favicon exists
+		if ff.verifyFaviconExists(faviconURL) {
+			return faviconURL, nil
+		}
+	}
+
+	// Try common favicon locations
+	faviconURL = ff.tryCommonLocations(parsedURL)
+	if faviconURL != "" {
+		return faviconURL, nil
+	}
+
+	// Fallback to Google's favicon service
+	return ff.getGoogleFavicon(parsedURL.Host), nil
+}
+
+// extractFromHTML fetches HTML content and extracts favicon URLs from head section
+func (ff *FaviconFetcher) extractFromHTML(targetURL string, baseURL *url.URL) (string, error) {
+	req, err := http.NewRequest("GET", targetURL, nil)
+	if err != nil {
+		return "", fmt.Errorf("failed to create request: %w", err)
+	}
+
+	// Set headers to mimic a real browser
+	req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36")
+	req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8")
+	req.Header.Set("Accept-Language", "en-US,en;q=0.9")
+	req.Header.Set("Accept-Encoding", "gzip, deflate, br")
+	req.Header.Set("Cache-Control", "no-cache")
+	req.Header.Set("Pragma", "no-cache")
+
+	resp, err := ff.client.Do(req)
+	if err != nil {
+		return "", fmt.Errorf("failed to fetch HTML: %w", err)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		return "", fmt.Errorf("HTTP %d: %s", resp.StatusCode, resp.Status)
+	}
+
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return "", fmt.Errorf("failed to read response body: %w", err)
+	}
+
+	content := string(body)
+
+	// Extract head section for faster processing
+	headContent := ff.extractHeadSection(content)
+
+	// Try to find favicon in head section
+	return ff.findFaviconInHead(headContent, baseURL), nil
+}
+
+// extractHeadSection extracts the <head> section from HTML content
+func (ff *FaviconFetcher) extractHeadSection(content string) string {
+	// Find head section with a more robust regex
+	headRegex := regexp.MustCompile(`(?is)<head[^>]*>(.*?)</head>`)
+	matches := headRegex.FindStringSubmatch(content)
+	if len(matches) > 1 {
+		return matches[1]
+	}
+
+	// Fallback: try to find from beginning to <body>
+	bodyRegex := regexp.MustCompile(`(?is)^.*?<body[^>]*>`)
+	matches = bodyRegex.FindStringSubmatch(content)
+	if len(matches) > 0 {
+		return matches[0]
+	}
+
+	// Last resort: return first 2000 characters
+	if len(content) > 2000 {
+		return content[:2000]
+	}
+	return content
+}
+
+// findFaviconInHead searches for favicon URLs in head section content
+func (ff *FaviconFetcher) findFaviconInHead(headContent string, baseURL *url.URL) string {
+	// Comprehensive favicon patterns in order of preference
+	patterns := []struct {
+		pattern  string
+		priority int
+	}{
+		// High priority: explicit favicon declarations
+		{`<link[^>]+rel=["'](?:icon|shortcut icon)["'][^>]+href=["']([^"']+)["']`, 1},
+		{`<link[^>]+href=["']([^"']+)["'][^>]+rel=["'](?:icon|shortcut icon)["']`, 1},
+
+		// Medium priority: Apple touch icons (usually higher quality)
+		{`<link[^>]+rel=["']apple-touch-icon["'][^>]+href=["']([^"']+)["']`, 2},
+		{`<link[^>]+href=["']([^"']+)["'][^>]+rel=["']apple-touch-icon["']`, 2},
+		{`<link[^>]+rel=["']apple-touch-icon-precomposed["'][^>]+href=["']([^"']+)["']`, 2},
+		{`<link[^>]+href=["']([^"']+)["'][^>]+rel=["']apple-touch-icon-precomposed["']`, 2},
+
+		// Lower priority: other icon types
+		{`<link[^>]+rel=["']android-chrome-[\w\-\d]+["'][^>]+href=["']([^"']+)["']`, 3},
+		{`<link[^>]+href=["']([^"']+)["'][^>]+rel=["']android-chrome-[\w\-\d]+["']`, 3},
+		{`<link[^>]+rel=["']mask-icon["'][^>]+href=["']([^"']+)["']`, 3},
+		{`<link[^>]+href=["']([^"']+)["'][^>]+rel=["']mask-icon["']`, 3},
+		{`<link[^>]+rel=["']fluid-icon["'][^>]+href=["']([^"']+)["']`, 3},
+		{`<link[^>]+href=["']([^"']+)["'][^>]+rel=["']fluid-icon["']`, 3},
+
+		// Meta tags that might contain icons
+		{`<meta[^>]+name=["']msapplication-TileImage["'][^>]+content=["']([^"']+)["']`, 4},
+
+		// Open Graph and Twitter images (can be used as fallback)
+		{`<meta[^>]+property=["']og:image["'][^>]+content=["']([^"']+)["']`, 5},
+		{`<meta[^>]+name=["']twitter:image["'][^>]+content=["']([^"']+)["']`, 5},
+
+		// Logo patterns
+		{`<link[^>]+rel=["']logo["'][^>]+href=["']([^"']+)["']`, 6},
+		{`<link[^>]+href=["']([^"']+)["'][^>]+rel=["']logo["']`, 6},
+
+		// Generic icon rel
+		{`<link[^>]+rel=["'][^"']*icon[^"']*["'][^>]+href=["']([^"']+)["']`, 7},
+		{`<link[^>]+href=["']([^"']+)["'][^>]+rel=["'][^"']*icon[^"']*["']`, 7},
+	}
+
+	var candidates []struct {
+		url      string
+		priority int
+	}
+
+	for _, p := range patterns {
+		re := regexp.MustCompile(p.pattern)
+		matches := re.FindAllStringSubmatch(headContent, -1)
+
+		for _, match := range matches {
+			if len(match) > 1 {
+				href := strings.TrimSpace(match[1])
+				if href != "" {
+					absoluteURL := ff.makeAbsoluteURL(href, baseURL)
+					candidates = append(candidates, struct {
+						url      string
+						priority int
+					}{url: absoluteURL, priority: p.priority})
+				}
+			}
+		}
+	}
+
+	// Return the highest priority candidate
+	if len(candidates) > 0 {
+		best := candidates[0]
+		for _, candidate := range candidates {
+			if candidate.priority < best.priority {
+				best = candidate
+			}
+		}
+		return best.url
+	}
+
+	return ""
+}
+
+// makeAbsoluteURL converts relative URLs to absolute URLs
+func (ff *FaviconFetcher) makeAbsoluteURL(href string, baseURL *url.URL) string {
+	// Remove any fragments
+	if idx := strings.Index(href, "#"); idx != -1 {
+		href = href[:idx]
+	}
+
+	// Handle different URL types
+	if strings.HasPrefix(href, "http://") || strings.HasPrefix(href, "https://") {
+		return href
+	}
+
+	if strings.HasPrefix(href, "//") {
+		return baseURL.Scheme + ":" + href
+	}
+
+	if strings.HasPrefix(href, "/") {
+		return baseURL.Scheme + "://" + baseURL.Host + href
+	}
+
+	// Relative path - construct proper URL
+	if baseURL.Path == "" || baseURL.Path == "/" {
+		return baseURL.Scheme + "://" + baseURL.Host + "/" + href
+	}
+
+	// Remove filename from base path
+	basePath := baseURL.Path
+	if lastSlash := strings.LastIndex(basePath, "/"); lastSlash != -1 {
+		basePath = basePath[:lastSlash+1]
+	}
+
+	return baseURL.Scheme + "://" + baseURL.Host + basePath + href
+}
+
+// tryCommonLocations tries common favicon file paths
+func (ff *FaviconFetcher) tryCommonLocations(baseURL *url.URL) string {
+	// Common favicon locations, ordered by likelihood
+	locations := []string{
+		"/favicon.ico",
+		"/favicon.png",
+		"/favicon.svg",
+		"/apple-touch-icon.png",
+		"/apple-touch-icon-precomposed.png",
+		"/android-chrome-192x192.png",
+		"/icon.png",
+		"/icon.svg",
+		"/logo.png",
+		"/logo.svg",
+		"/assets/favicon.ico",
+		"/assets/favicon.png",
+		"/assets/icon.png",
+		"/static/favicon.ico",
+		"/static/favicon.png",
+		"/static/icon.png",
+		"/images/favicon.ico",
+		"/images/favicon.png",
+		"/img/favicon.ico",
+		"/img/favicon.png",
+		"/favicon-32x32.png",
+		"/favicon-16x16.png",
+		"/icon-192x192.png",
+		"/icon-512x512.png",
+	}
+
+	for _, path := range locations {
+		faviconURL := baseURL.Scheme + "://" + baseURL.Host + path
+
+		if ff.verifyFaviconExists(faviconURL) {
+			return faviconURL
+		}
+	}
+
+	return ""
+}
+
+// verifyFaviconExists checks if a favicon URL exists and is accessible
+func (ff *FaviconFetcher) verifyFaviconExists(faviconURL string) bool {
+	req, err := http.NewRequest("HEAD", faviconURL, nil)
+	if err != nil {
+		return false
+	}
+
+	req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36")
+
+	resp, err := ff.client.Do(req)
+	if err != nil {
+		return false
+	}
+	defer resp.Body.Close()
+
+	// Check if the response is successful and contains an image
+	if resp.StatusCode == http.StatusOK {
+		contentType := resp.Header.Get("Content-Type")
+		return strings.HasPrefix(contentType, "image/") ||
+			strings.HasSuffix(faviconURL, ".ico") ||
+			strings.HasSuffix(faviconURL, ".png") ||
+			strings.HasSuffix(faviconURL, ".svg") ||
+			strings.HasSuffix(faviconURL, ".jpg") ||
+			strings.HasSuffix(faviconURL, ".jpeg") ||
+			strings.HasSuffix(faviconURL, ".gif") ||
+			strings.HasSuffix(faviconURL, ".webp")
+	}
+
+	return false
+}
+
+// getGoogleFavicon returns Google's favicon service URL as fallback
+func (ff *FaviconFetcher) getGoogleFavicon(domain string) string {
+	// Try different sizes for better quality
+	return fmt.Sprintf("https://www.google.com/s2/favicons?domain=%s&sz=128", domain)
+}
+
+// FetchMultipleFavicons fetches multiple favicon candidates for a URL
+func (ff *FaviconFetcher) FetchMultipleFavicons(targetURL string, maxResults int) []string {
+	parsedURL, err := url.Parse(targetURL)
+	if err != nil {
+		return []string{ff.getGoogleFavicon("example.com")}
+	}
+
+	var favicons []string
+
+	// Try HTML extraction
+	if htmlFavicon, err := ff.extractFromHTML(targetURL, parsedURL); err == nil && htmlFavicon != "" {
+		if ff.verifyFaviconExists(htmlFavicon) {
+			favicons = append(favicons, htmlFavicon)
+		}
+	}
+
+	// Try common locations
+	locations := []string{
+		"/favicon.ico", "/favicon.png", "/favicon.svg",
+		"/apple-touch-icon.png", "/icon.png", "/logo.png",
+		"/assets/favicon.ico", "/static/favicon.ico", "/images/favicon.ico",
+	}
+
+	for _, path := range locations {
+		faviconURL := parsedURL.Scheme + "://" + parsedURL.Host + path
+		if ff.verifyFaviconExists(faviconURL) && !containsString(favicons, faviconURL) {
+			favicons = append(favicons, faviconURL)
+			if len(favicons) >= maxResults {
+				break
+			}
+		}
+	}
+
+	// Add Google fallback if no favicons found or if we want more results
+	if len(favicons) == 0 || len(favicons) < maxResults {
+		googleFavicon := ff.getGoogleFavicon(parsedURL.Host)
+		if !containsString(favicons, googleFavicon) {
+			favicons = append(favicons, googleFavicon)
+		}
+	}
+
+	return favicons
+}
+
+// containsString checks if a string slice contains a specific string
+func containsString(slice []string, item string) bool {
+	for _, s := range slice {
+		if s == item {
+			return true
+		}
+	}
+	return false
+}
+
+// Global instance
+var faviconFetcher = NewFaviconFetcher()
+
+// GetFavicon fetches the best favicon for a URL (convenience function)
+func GetFavicon(url string) (string, error) {
+	return faviconFetcher.FetchFavicon(url)
+}
+
+// GetAllFavicons fetches multiple favicon candidates for a URL
+func GetAllFavicons(url string, maxResults int) []string {
+	return faviconFetcher.FetchMultipleFavicons(url, maxResults)
+}
@@ -0,0 +1,162 @@
+package services
+
+import (
+	"net/url"
+	"strings"
+	"testing"
+)
+
+func TestFaviconFetcher(t *testing.T) {
+	fetcher := NewFaviconFetcher()
+
+	// Test cases with different types of websites
+	testCases := []struct {
+		name     string
+		url      string
+		expected string // We'll just check that we get some result
+	}{
+		{
+			name:     "GitHub",
+			url:      "https://github.com",
+			expected: "", // We expect to find a favicon
+		},
+		{
+			name:     "Google",
+			url:      "https://www.google.com",
+			expected: "", // We expect to find a favicon
+		},
+		{
+			name:     "Stack Overflow",
+			url:      "https://stackoverflow.com",
+			expected: "", // We expect to find a favicon
+		},
+		{
+			name:     "Reddit",
+			url:      "https://www.reddit.com",
+			expected: "", // We expect to find a favicon
+		},
+		{
+			name:     "Wikipedia",
+			url:      "https://en.wikipedia.org",
+			expected: "", // We expect to find a favicon
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			favicon, err := fetcher.FetchFavicon(tc.url)
+
+			if err != nil {
+				t.Logf("Warning: Could not fetch favicon for %s: %v", tc.name, err)
+				// Don't fail the test, as some sites might block requests
+				return
+			}
+
+			if favicon == "" {
+				t.Errorf("Expected to find a favicon for %s, but got empty string", tc.name)
+				return
+			}
+
+			t.Logf("✓ Found favicon for %s: %s", tc.name, favicon)
+		})
+	}
+}
+
+func TestMultipleFavicons(t *testing.T) {
+	fetcher := NewFaviconFetcher()
+
+	testURL := "https://github.com"
+	favicons := fetcher.FetchMultipleFavicons(testURL, 5)
+
+	if len(favicons) == 0 {
+		t.Error("Expected to find at least one favicon")
+	}
+
+	t.Logf("Found %d favicons for %s:", len(favicons), testURL)
+	for i, favicon := range favicons {
+		t.Logf("  %d. %s", i+1, favicon)
+	}
+}
+
+func TestMakeAbsoluteURL(t *testing.T) {
+	fetcher := NewFaviconFetcher()
+
+	baseURL, _ := url.Parse("https://example.com/path/page.html")
+
+	testCases := []struct {
+		input    string
+		expected string
+	}{
+		{
+			input:    "/favicon.ico",
+			expected: "https://example.com/favicon.ico",
+		},
+		{
+			input:    "favicon.ico",
+			expected: "https://example.com/path/favicon.ico",
+		},
+		{
+			input:    "../favicon.ico",
+			expected: "https://example.com/favicon.ico",
+		},
+		{
+			input:    "https://cdn.example.com/favicon.ico",
+			expected: "https://cdn.example.com/favicon.ico",
+		},
+		{
+			input:    "//cdn.example.com/favicon.ico",
+			expected: "https://cdn.example.com/favicon.ico",
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.input, func(t *testing.T) {
+			result := fetcher.makeAbsoluteURL(tc.input, baseURL)
+			if result != tc.expected {
+				t.Errorf("Expected %s, got %s", tc.expected, result)
+			}
+		})
+	}
+}
+
+func TestExtractHeadSection(t *testing.T) {
+	fetcher := NewFaviconFetcher()
+
+	htmlContent := `<!DOCTYPE html>
+<html>
+<head>
+    <title>Test Page</title>
+    <link rel="icon" href="/favicon.ico">
+    <meta name="description" content="Test description">
+</head>
+<body>
+    <h1>Test Content</h1>
+</body>
+</html>`
+
+	headContent := fetcher.extractHeadSection(htmlContent)
+
+	// Should contain the favicon link
+	if !strings.Contains(headContent, `<link rel="icon" href="/favicon.ico">`) {
+		t.Error("Expected to find favicon link in extracted head section")
+	}
+
+	// Should not contain body content
+	if strings.Contains(headContent, "<h1>Test Content</h1>") {
+		t.Error("Expected head section to not contain body content")
+	}
+}
+
+// Benchmark tests
+func BenchmarkFaviconFetch(b *testing.B) {
+	fetcher := NewFaviconFetcher()
+	url := "https://github.com"
+
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, err := fetcher.FetchFavicon(url)
+		if err != nil {
+			b.Logf("Error fetching favicon: %v", err)
+		}
+	}
+}
@@ -25,7 +25,7 @@ type WebsiteMetadata struct {
 // FetchWebsiteMetadata extracts metadata from a URL
 func FetchWebsiteMetadata(targetURL string) (*WebsiteMetadata, error) {
 	// Parse URL to ensure it's valid
-	parsedURL, err := url.Parse(targetURL)
+	_, err := url.Parse(targetURL)
 	if err != nil {
 		return nil, fmt.Errorf("invalid URL: %w", err)
 	}
@@ -69,14 +69,11 @@ func FetchWebsiteMetadata(targetURL string) (*WebsiteMetadata, error) {
 	metadata = extractTwitterMetadata(content, metadata)
 	metadata = extractBasicHTMLMetadata(content, metadata)

-	// Extract favicon
+	// Extract favicon using enhanced fetcher
 	if metadata.Favicon == "" {
-		metadata.Favicon = extractFavicon(content, parsedURL)
-	}
-
-	// If still no favicon, try default locations
-	if metadata.Favicon == "" {
-		metadata.Favicon = getDefaultFavicon(parsedURL)
+		if favicon, err := GetFavicon(targetURL); err == nil && favicon != "" {
+			metadata.Favicon = favicon
+		}
 	}

 	return metadata, nil
@@ -0,0 +1,76 @@
+package main
+
+import (
+	"fmt"
+	"os"
+	"time"
+
+	"github.com/trackeep/backend/services"
+)
+
+func main() {
+	if len(os.Args) < 2 {
+		fmt.Println("Usage: go run favicon_test.go <URL>")
+		fmt.Println("Example: go run favicon_test.go https://github.com")
+		os.Exit(1)
+	}
+
+	url := os.Args[1]
+	fmt.Printf("Testing favicon fetching for: %s\n\n", url)
+
+	// Test the enhanced favicon fetcher
+	fetcher := services.NewFaviconFetcher()
+
+	fmt.Println("=== Enhanced Favicon Fetcher ===")
+	start := time.Now()
+	favicon, err := fetcher.FetchFavicon(url)
+	duration := time.Since(start)
+
+	if err != nil {
+		fmt.Printf("❌ Error: %v\n", err)
+	} else if favicon == "" {
+		fmt.Printf("❌ No favicon found\n")
+	} else {
+		fmt.Printf("✅ Favicon found: %s (took %v)\n", favicon, duration)
+	}
+
+	fmt.Println("\n=== Multiple Favicon Candidates ===")
+	start = time.Now()
+	favicons := fetcher.FetchMultipleFavicons(url, 5)
+	duration = time.Since(start)
+
+	fmt.Printf("Found %d favicon candidates (took %v):\n", len(favicons), duration)
+	for i, f := range favicons {
+		fmt.Printf("  %d. %s\n", i+1, f)
+	}
+
+	fmt.Println("\n=== Original Metadata Service ===")
+	start = time.Now()
+	metadata, err := services.FetchWebsiteMetadata(url)
+	duration = time.Since(start)
+
+	if err != nil {
+		fmt.Printf("❌ Error: %v\n", err)
+	} else {
+		fmt.Printf("✅ Metadata fetched (took %v):\n", duration)
+		fmt.Printf("  Title: %s\n", metadata.Title)
+		fmt.Printf("  Description: %s\n", metadata.Description)
+		fmt.Printf("  Favicon: %s\n", metadata.Favicon)
+		fmt.Printf("  Site Name: %s\n", metadata.SiteName)
+	}
+
+	fmt.Println("\n=== Comparison ===")
+	if favicon != "" && metadata != nil && metadata.Favicon != "" {
+		if favicon == metadata.Favicon {
+			fmt.Println("✅ Both methods returned the same favicon")
+		} else {
+			fmt.Println("⚠️  Different favicons returned:")
+			fmt.Printf("  Enhanced: %s\n", favicon)
+			fmt.Printf("  Original: %s\n", metadata.Favicon)
+		}
+	} else if metadata == nil {
+		fmt.Println("⚠️  Original metadata service failed, enhanced method succeeded")
+	} else {
+		fmt.Println("⚠️  Could not compare favicon results")
+	}
+}