From b6f40af67fef89a0cf38e9d1d3a3c2738e41c159 Mon Sep 17 00:00:00 2001 From: Tomas Dvorak Date: Fri, 8 May 2026 11:07:34 +0200 Subject: [PATCH] feat(hub): improve WHOIS lookup reliability and enhance site UI Implement enhanced WHOIS lookup strategies, specifically targeting .eu domains through EURid web scraping and alternative services to improve data accuracy for expiry dates. - Add EURid web scraping and alternative WHOIS service support for .eu domains - Increase timeouts for .eu domain lookups in TCP and native WHOIS - Improve domain scheduler to prevent overwriting valid data with zero-value dates - Enhance site UI with subdomain indicators in domain tables - Add filtering capabilities to the calendar view - Implement drag-and-drop reordering for systems table - Add new debug and test utilities for WHOIS and date parsing logic --- internal/hub/domains/scheduler.go | 6 +- internal/hub/domains/whois/lookup.go | 464 +++++++++++++++++- internal/hub/monitors/api.go | 6 +- .../src/components/calendar/calendar-view.tsx | 154 +++++- .../domains-table/domains-table.tsx | 98 ++-- .../monitors-table/monitors-table.tsx | 10 +- .../site/src/components/routes/domain.tsx | 392 ++++++++++++--- .../systems-table/systems-table.tsx | 216 ++++++-- test/alternative_whois_debug.go | 160 ++++++ test/date_parsing_demo.go | 94 ++++ test/eurid_scrape_debug.go | 112 +++++ test/eurid_scrape_improved.go | 163 ++++++ test/tcp_whois_debug.go | 112 +++++ test/test_whois.go | 119 +++++ test/whoisxml_test.js | 23 + 15 files changed, 1934 insertions(+), 195 deletions(-) create mode 100644 test/alternative_whois_debug.go create mode 100644 test/date_parsing_demo.go create mode 100644 test/eurid_scrape_debug.go create mode 100644 test/eurid_scrape_improved.go create mode 100644 test/tcp_whois_debug.go create mode 100644 test/test_whois.go create mode 100644 test/whoisxml_test.js diff --git a/internal/hub/domains/scheduler.go b/internal/hub/domains/scheduler.go index 59bbb2f..6533b3b 100644 --- a/internal/hub/domains/scheduler.go +++ b/internal/hub/domains/scheduler.go @@ -151,13 +151,13 @@ func (s *Scheduler) checkDomain(record *core.Record) error { oldRecord := record.Fresh() // Update record (only overwrite if new data is present to preserve valid data on partial lookups) - if newData.ExpiryDate != nil { + if newData.ExpiryDate != nil && newData.ExpiryDate.After(time.Time{}) { record.Set("expiry_date", *newData.ExpiryDate) } - if newData.CreationDate != nil { + if newData.CreationDate != nil && newData.CreationDate.After(time.Time{}) { record.Set("creation_date", *newData.CreationDate) } - if newData.UpdatedDate != nil { + if newData.UpdatedDate != nil && newData.UpdatedDate.After(time.Time{}) { record.Set("updated_date", *newData.UpdatedDate) } if newData.RegistrarName != "" { diff --git a/internal/hub/domains/whois/lookup.go b/internal/hub/domains/whois/lookup.go index 2689021..d5b22df 100644 --- a/internal/hub/domains/whois/lookup.go +++ b/internal/hub/domains/whois/lookup.go @@ -9,6 +9,7 @@ import ( "encoding/hex" "encoding/json" "fmt" + "io" "net" "net/http" "os/exec" @@ -79,32 +80,55 @@ func (s *LookupService) LookupDomain(ctx context.Context, domainName string) (*d func (s *LookupService) LookupWHOIS(ctx context.Context, domainName string) (*domain.WHOISData, error) { var lastErr error - // Try RDAP first (modern replacement for WHOIS) + // Try RDAP first data, err := s.tryRDAP(ctx, domainName) if err == nil && data != nil && hasValidData(data) { return data, nil } - lastErr = err + if err != nil { + lastErr = err + } - // Try pure-Go TCP WHOIS (works in containers without whois binary) + // Try TCP WHOIS (this should work for .eu domains) data, err = s.tryTCPWHOIS(ctx, domainName) if err == nil && data != nil && hasValidData(data) { return data, nil } - if lastErr == nil { + if err != nil { lastErr = err } - // Try native whois command + // Try native whois command (often works when TCP fails) data, err = s.tryNativeWHOIS(ctx, domainName) if err == nil && data != nil && hasValidData(data) { return data, nil } - if lastErr == nil { + if err != nil { lastErr = err } - // Try WhoisXML API if key is configured + // Try EURid web scraping for .eu domains to get expiry dates + parts := strings.Split(domainName, ".") + if len(parts) >= 2 && strings.ToLower(parts[len(parts)-1]) == "eu" { + data, err = s.tryEURidWebScraping(ctx, domainName) + if err == nil && data != nil && hasValidData(data) { + return data, nil + } + if err != nil { + lastErr = err + } + + // Try alternative WHOIS services for .eu domains + data, err = s.tryAlternativeWHOIS(ctx, domainName) + if err == nil && data != nil && hasValidData(data) { + return data, nil + } + if err != nil { + lastErr = err + } + } + + // Try WhoisXML API if key is configured (this can provide expiry dates for .eu domains) if s.whoisXMLAPIKey != "" { data, err = s.tryWhoisXML(ctx, domainName) if err == nil && data != nil { @@ -251,8 +275,15 @@ func (s *LookupService) tryNativeWHOIS(ctx context.Context, domainName string) ( return nil, fmt.Errorf("whois command not found") } + // Use longer timeout for .eu domains + timeout := 10 * time.Second + parts := strings.Split(domainName, ".") + if len(parts) >= 2 && strings.ToLower(parts[len(parts)-1]) == "eu" { + timeout = 20 * time.Second + } + // Execute whois with timeout - cmdCtx, cancel := context.WithTimeout(ctx, 10*time.Second) + cmdCtx, cancel := context.WithTimeout(ctx, timeout) defer cancel() cmd := exec.CommandContext(cmdCtx, "whois", domainName) @@ -312,7 +343,13 @@ func (s *LookupService) tryTCPWHOIS(ctx context.Context, domainName string) (*do addr := net.JoinHostPort(server, "43") - dialer := &net.Dialer{Timeout: 10 * time.Second} + // Use longer timeout for .eu domains as they can be slow + timeout := 10 * time.Second + if tld == "eu" { + timeout = 20 * time.Second + } + + dialer := &net.Dialer{Timeout: timeout} conn, err := dialer.DialContext(ctx, "tcp", addr) if err != nil { return nil, fmt.Errorf("tcp whois dial failed: %w", err) @@ -326,7 +363,7 @@ func (s *LookupService) tryTCPWHOIS(ctx context.Context, domainName string) (*do } // Read response with deadline - if err := conn.SetReadDeadline(time.Now().Add(10 * time.Second)); err != nil { + if err := conn.SetReadDeadline(time.Now().Add(timeout)); err != nil { return nil, err } @@ -414,11 +451,417 @@ func (s *LookupService) tryWhoisXML(ctx context.Context, domainName string) (*do }, nil } +// parseEUWHOIS parses .eu domain WHOIS output which has a unique format +func (s *LookupService) parseEUWHOIS(output, domainName string) (*domain.WHOISData, error) { + lines := strings.Split(output, "\n") + + var registrarName, organization string + var statuses []string + + // Parse the .eu specific format + currentSection := "" + for _, line := range lines { + line = strings.TrimSpace(line) + + // Skip comments and empty lines + if line == "" || strings.HasPrefix(line, "%") { + continue + } + + // Track sections + if strings.HasPrefix(line, "Registrant:") { + currentSection = "registrant" + continue + } + if strings.HasPrefix(line, "Technical:") { + currentSection = "technical" + continue + } + if strings.HasPrefix(line, "Registrar:") { + currentSection = "registrar" + continue + } + if strings.HasPrefix(line, "Name servers:") { + currentSection = "nameservers" + continue + } + + // Parse based on current section + if idx := strings.Index(line, ":"); idx > 0 { + key := strings.TrimSpace(line[:idx]) + value := strings.TrimSpace(line[idx+1:]) + + switch currentSection { + case "registrar": + if strings.TrimSpace(key) == "Name" { + registrarName = value + } + if strings.TrimSpace(key) == "Website" { + // Could extract website URL if needed + } + case "technical": + if strings.TrimSpace(key) == "Organisation" { + organization = value + } + } + } + } + + // For .eu domains, we often don't get expiry dates, so we'll return what we have + return &domain.WHOISData{ + DomainName: domainName, + Status: statuses, + DNSSEC: "", // .eu WHOIS doesn't provide DNSSEC info + Dates: domain.WHOISDates{ + ExpiryDate: nil, // .eu domains don't show expiry in TCP WHOIS + CreationDate: nil, + UpdatedDate: nil, + }, + Registrar: domain.WHOISRegistrar{ + Name: registrarName, + ID: "", + URL: "", + }, + Registrant: domain.WHOISContact{ + Name: "NOT DISCLOSED", + Organization: organization, + }, + }, nil +} + +// tryEURidWebScraping attempts to scrape EURid's web WHOIS for .eu domains +func (s *LookupService) tryEURidWebScraping(ctx context.Context, domainName string) (*domain.WHOISData, error) { + // Try multiple EURid endpoints + endpoints := []string{ + fmt.Sprintf("https://whois.eurid.eu/en/?q=%s", domainName), + fmt.Sprintf("https://whois.eurid.eu/en/search?q=%s", domainName), + fmt.Sprintf("https://www.eurid.eu/en/whois/?domain=%s", domainName), + } + + for _, url := range endpoints { + data, err := s.tryEURidEndpoint(ctx, url, domainName) + if err == nil && data != nil { + return data, nil + } + } + + return nil, fmt.Errorf("all EURid web scraping attempts failed for %s", domainName) +} + +// tryEURidEndpoint attempts to scrape a specific EURid endpoint +func (s *LookupService) tryEURidEndpoint(ctx context.Context, url, domainName string) (*domain.WHOISData, error) { + req, err := http.NewRequestWithContext(ctx, "GET", url, nil) + if err != nil { + return nil, fmt.Errorf("failed to create request: %w", err) + } + + // Use more realistic browser headers + req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36") + req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7") + req.Header.Set("Accept-Language", "en-US,en;q=0.9") + req.Header.Set("Accept-Encoding", "gzip, deflate, br") + req.Header.Set("Cache-Control", "no-cache") + req.Header.Set("Pragma", "no-cache") + req.Header.Set("Sec-Ch-Ua", "\"Not_A Brand\";v=\"8\", \"Chromium\";v=\"120\", \"Google Chrome\";v=\"120\"") + req.Header.Set("Sec-Ch-Ua-Mobile", "?0") + req.Header.Set("Sec-Ch-Ua-Platform", "\"Windows\"") + req.Header.Set("Sec-Fetch-Dest", "document") + req.Header.Set("Sec-Fetch-Mode", "navigate") + req.Header.Set("Sec-Fetch-Site", "none") + req.Header.Set("Sec-Fetch-User", "?1") + req.Header.Set("Upgrade-Insecure-Requests", "1") + + client := &http.Client{Timeout: 20 * time.Second} + resp, err := client.Do(req) + if err != nil { + return nil, fmt.Errorf("failed to fetch EURid web page: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != 200 { + return nil, fmt.Errorf("EURid web page returned status %d", resp.StatusCode) + } + + // Read the HTML response + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("failed to read EURid response: %w", err) + } + + // Parse the HTML to extract expiry date + return s.parseEURidWebHTML(string(body), domainName) +} + +// tryAlternativeWHOIS tries alternative WHOIS services for .eu domains +func (s *LookupService) tryAlternativeWHOIS(ctx context.Context, domainName string) (*domain.WHOISData, error) { + // Try multiple alternative WHOIS services + services := []struct { + name string + url string + }{ + {"whois.com", fmt.Sprintf("https://www.whois.com/whois/%s", domainName)}, + {"who.is", fmt.Sprintf("https://who.is/whois/%s", domainName)}, + {"ip2location.com", fmt.Sprintf("https://www.ip2location.com/whois/%s", domainName)}, + } + + for _, service := range services { + data, err := s.tryAlternativeWHOISService(ctx, service.name, service.url, domainName) + if err == nil && data != nil { + return data, nil + } + } + + return nil, fmt.Errorf("all alternative WHOIS services failed for %s", domainName) +} + +// tryAlternativeWHOISService attempts to fetch WHOIS data from an alternative service +func (s *LookupService) tryAlternativeWHOISService(ctx context.Context, serviceName, url, domainName string) (*domain.WHOISData, error) { + req, err := http.NewRequestWithContext(ctx, "GET", url, nil) + if err != nil { + return nil, fmt.Errorf("failed to create request for %s: %w", serviceName, err) + } + + req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36") + req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8") + + client := &http.Client{Timeout: 15 * time.Second} + resp, err := client.Do(req) + if err != nil { + return nil, fmt.Errorf("failed to fetch %s: %w", serviceName, err) + } + defer resp.Body.Close() + + if resp.StatusCode != 200 { + return nil, fmt.Errorf("%s returned status %d", serviceName, resp.StatusCode) + } + + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("failed to read %s response: %w", serviceName, err) + } + + return s.parseAlternativeWHOISHTML(string(body), domainName, serviceName) +} + +// parseAlternativeWHOISHTML parses HTML from alternative WHOIS services +func (s *LookupService) parseAlternativeWHOISHTML(html, domainName, serviceName string) (*domain.WHOISData, error) { + var expiryDate, registrarName, status string + + // Look for expiry date patterns (common across WHOIS services) + expiryPatterns := []string{ + `Expiry Date:\s*]*>\s*([^<\n]+)`, + `Expiry Date:]*>\s*([^<\n]+)`, + `Expires on:\s*]*>\s*([^<\n]+)`, + `Expires:\s*]*>\s*([^<\n]+)`, + `"expiry":"([^"]+)"`, + `"expires":"([^"]+)"`, + `data-expiry="([^"]+)"`, + `\d{4}-\d{2}-\d{2}`, // ISO date pattern + `\d{2}/\d{2}/\d{4}`, // DD/MM/YYYY pattern + } + + for _, pattern := range expiryPatterns { + re := regexp.MustCompile(pattern) + matches := re.FindStringSubmatch(html) + if len(matches) > 1 { + expiryDate = strings.TrimSpace(matches[1]) + break + } + } + + // Look for registrar name + registrarPatterns := []string{ + `Registrar:\s*]*>\s*([^<\n]+)`, + `Registrar:]*>\s*([^<\n]+)`, + `Registered through:\s*]*>\s*([^<\n]+)`, + `"registrar":"([^"]+)"`, + `data-registrar="([^"]+)"`, + } + + for _, pattern := range registrarPatterns { + re := regexp.MustCompile(pattern) + matches := re.FindStringSubmatch(html) + if len(matches) > 1 { + registrarName = strings.TrimSpace(matches[1]) + break + } + } + + // Look for status + statusPatterns := []string{ + `Status:\s*]*>\s*([^<\n]+)`, + `Status:]*>\s*([^<\n]+)`, + `"status":"([^"]+)"`, + `data-status="([^"]+)"`, + } + + for _, pattern := range statusPatterns { + re := regexp.MustCompile(pattern) + matches := re.FindStringSubmatch(html) + if len(matches) > 1 { + status = strings.TrimSpace(matches[1]) + break + } + } + + // Parse expiry date if found + var parsedExpiry *time.Time + if expiryDate != "" { + // Try different date formats + dateFormats := []string{ + "2006-01-02", // ISO + "02/01/2006", // DD/MM/YYYY + "01/02/2006", // MM/DD/YYYY + "2006-01-02T15:04:05Z", // ISO with time + } + + for _, format := range dateFormats { + if parsed, err := time.Parse(format, expiryDate); err == nil { + parsedExpiry = &parsed + break + } + } + } + + // Create WHOIS data structure + var statuses []string + if status != "" { + statuses = []string{status} + } + + return &domain.WHOISData{ + DomainName: domainName, + Status: statuses, + DNSSEC: "", + Dates: domain.WHOISDates{ + ExpiryDate: parsedExpiry, + CreationDate: nil, + UpdatedDate: nil, + }, + Registrar: domain.WHOISRegistrar{ + Name: registrarName, + ID: "", + URL: "", + }, + Registrant: domain.WHOISContact{ + Name: "NOT DISCLOSED", + Organization: "", + }, + }, nil +} + +// parseEURidWebHTML parses EURid's web WHOIS HTML to extract domain information +func (s *LookupService) parseEURidWebHTML(html, domainName string) (*domain.WHOISData, error) { + // This is a simplified HTML parser - in production, you'd want to use a proper HTML parser + // For now, we'll use regex to find key information + + var expiryDate, registrarName, status string + + // Look for expiry date patterns in EURid's HTML + expiryPatterns := []string{ + `Expiry date:\s*\s*(\d{2}/\d{2}/\d{4})`, + `Expiry date:\s*(\d{2}/\d{2}/\d{4})`, + `Expiry date\s*(\d{2}/\d{2}/\d{4})`, + `"expiryDate":"([^"]+)"`, + `data-expiry="([^"]+)"`, + } + + for _, pattern := range expiryPatterns { + re := regexp.MustCompile(pattern) + matches := re.FindStringSubmatch(html) + if len(matches) > 1 { + expiryDate = matches[1] + break + } + } + + // Look for registrar name + registrarPatterns := []string{ + `Registrar:\s*\s*([^<\n]+)`, + `Registrar:\s*([^<\n]+)`, + `Registrar\s*([^<\n]+)`, + `"registrar":"([^"]+)"`, + `data-registrar="([^"]+)"`, + } + + for _, pattern := range registrarPatterns { + re := regexp.MustCompile(pattern) + matches := re.FindStringSubmatch(html) + if len(matches) > 1 { + registrarName = strings.TrimSpace(matches[1]) + break + } + } + + // Look for status + statusPatterns := []string{ + `Status:\s*\s*([^<\n]+)`, + `Status:\s*([^<\n]+)`, + `Status\s*([^<\n]+)`, + `"status":"([^"]+)"`, + `data-status="([^"]+)"`, + } + + for _, pattern := range statusPatterns { + re := regexp.MustCompile(pattern) + matches := re.FindStringSubmatch(html) + if len(matches) > 1 { + status = strings.TrimSpace(matches[1]) + break + } + } + + // Parse expiry date if found + var parsedExpiry *time.Time + if expiryDate != "" { + // Try DD/MM/YYYY format first + if parsed, err := time.Parse("02/01/2006", expiryDate); err == nil { + parsedExpiry = &parsed + } else if parsed, err := time.Parse("2006-01-02", expiryDate); err == nil { + parsedExpiry = &parsed + } + } + + // Create WHOIS data structure + var statuses []string + if status != "" { + statuses = []string{status} + } + + return &domain.WHOISData{ + DomainName: domainName, + Status: statuses, + DNSSEC: "", + Dates: domain.WHOISDates{ + ExpiryDate: parsedExpiry, + CreationDate: nil, + UpdatedDate: nil, + }, + Registrar: domain.WHOISRegistrar{ + Name: registrarName, + ID: "", + URL: "", + }, + Registrant: domain.WHOISContact{ + Name: "NOT DISCLOSED", + Organization: "", + }, + }, nil +} + // parseWHOISOutput parses the raw WHOIS text output func (s *LookupService) parseWHOISOutput(output, domainName string) (*domain.WHOISData, error) { lines := strings.Split(output, "\n") data := make(map[string]string) + // Special handling for .eu domains which have a different format + parts := strings.Split(domainName, ".") + isEUDomain := len(parts) >= 2 && strings.ToLower(parts[len(parts)-1]) == "eu" + + if isEUDomain { + return s.parseEUWHOIS(output, domainName) + } + for _, line := range lines { line = strings.TrimSpace(line) if line == "" || strings.HasPrefix(line, "%") { @@ -465,6 +908,7 @@ func (s *LookupService) parseWHOISOutput(output, domainName string) (*domain.WHO ) // Extract registrar - try multiple field names used by different WHOIS servers + // .eu domains use "Name:" under Registrar section registrarName := data["registrar"] if registrarName == "" { registrarName = data["registrar_name"] diff --git a/internal/hub/monitors/api.go b/internal/hub/monitors/api.go index 04624d5..e0dceb0 100644 --- a/internal/hub/monitors/api.go +++ b/internal/hub/monitors/api.go @@ -293,8 +293,12 @@ func (h *APIHandler) createMonitor(e *core.RequestEvent) error { h.scheduler.AddMonitor(record) // Run initial check synchronously so the monitor shows real status immediately - if _, err := h.scheduler.RunManualCheck(record.Id); err != nil { + result, err := h.scheduler.RunManualCheck(record.Id) + if err != nil { log.Printf("[monitor-api] Initial check failed for %s: %v", record.Id, err) + // Note: The monitor will remain in pending status and the scheduler will retry + } else { + log.Printf("[monitor-api] Initial check completed for %s: status=%s, ping=%v", record.Id, result.Status, result.Ping) } // Re-fetch the updated record to get the new status diff --git a/internal/site/src/components/calendar/calendar-view.tsx b/internal/site/src/components/calendar/calendar-view.tsx index 35f4425..067e819 100644 --- a/internal/site/src/components/calendar/calendar-view.tsx +++ b/internal/site/src/components/calendar/calendar-view.tsx @@ -5,11 +5,25 @@ import { useQuery } from "@tanstack/react-query" import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card" import { Button } from "@/components/ui/button" import { Link } from "@/components/router" -import { ChevronLeft, ChevronRight, Calendar as CalendarIcon, AlertCircle, Globe, Shield } from "lucide-react" +import { ChevronLeft, ChevronRight, Calendar as CalendarIcon, AlertCircle, Globe, Shield, Filter, X } from "lucide-react" import { getCalendarEvents, type CalendarEvent } from "@/lib/incidents" +import { + DropdownMenu, + DropdownMenuCheckboxItem, + DropdownMenuContent, + DropdownMenuItem, + DropdownMenuLabel, + DropdownMenuSeparator, + DropdownMenuTrigger, +} from "@/components/ui/dropdown-menu" export function CalendarView() { const [currentDate, setCurrentDate] = useState(new Date()) + const [eventFilters, setEventFilters] = useState({ + domain_expiry: true, + ssl_expiry: true, + incident: true, + }) const year = currentDate.getFullYear() const month = currentDate.getMonth() @@ -46,20 +60,22 @@ export function CalendarView() { // Days of month for (let day = 1; day <= daysInMonth; day++) { const dateStr = `${year}-${String(month + 1).padStart(2, "0")}-${String(day).padStart(2, "0")}` - const dayEvents = events?.filter((e) => e.date === dateStr) || [] + const dayEvents = events?.filter((e) => + e.date === dateStr && eventFilters[e.type as keyof typeof eventFilters] + ) || [] d.push({ day, events: dayEvents }) } return d - }, [year, month, daysInMonth, firstDayOfMonth, events]) + }, [year, month, daysInMonth, firstDayOfMonth, events, eventFilters]) const upcomingEvents = useMemo(() => { const today = toDateString(new Date()) return (events || []) - .filter((event) => event.date >= today) + .filter((event) => event.date >= today && eventFilters[event.type as keyof typeof eventFilters]) .sort((a, b) => a.date.localeCompare(b.date)) .slice(0, 8) - }, [events]) + }, [events, eventFilters]) const prevMonth = () => { setCurrentDate(new Date(year, month - 1, 1)) @@ -123,26 +139,116 @@ export function CalendarView() { return ( -
- -
- +
+ {/* Title Row */} +
+ +
+ +
+ Calendar View + +
+ + + + {monthNames[month]} {year} + + +
+
+ + {/* Filter Controls Row */} +
+
+ Show: +
+ + + +
+
+
+ + + + + + Event Types + + setEventFilters(prev => ({ ...prev, domain_expiry: checked }))} + > +
+ + Domain Expiry +
+ + setEventFilters(prev => ({ ...prev, ssl_expiry: checked }))} + > +
+ + SSL Expiry +
+ + setEventFilters(prev => ({ ...prev, incident: checked }))} + > +
+ + Incidents +
+ + + setEventFilters({ domain_expiry: true, ssl_expiry: true, incident: true })}> + Show All + + setEventFilters({ domain_expiry: true, ssl_expiry: false, incident: false })}> + Domain Only + + setEventFilters({ domain_expiry: false, ssl_expiry: true, incident: false })}> + SSL Only + + +
- Calendar View - -
- - - - {monthNames[month]} {year} - -
diff --git a/internal/site/src/components/domains-table/domains-table.tsx b/internal/site/src/components/domains-table/domains-table.tsx index 8e4dddd..fa0c462 100644 --- a/internal/site/src/components/domains-table/domains-table.tsx +++ b/internal/site/src/components/domains-table/domains-table.tsx @@ -41,16 +41,15 @@ import { DropdownMenuTrigger, DropdownMenuCheckboxItem, } from "@/components/ui/dropdown-menu" -import { Badge } from "@/components/ui/badge" import { Input } from "@/components/ui/input" import { getDomains, deleteDomain, refreshDomain, - getStatusBadgeColor, - getStatusLabel, + getDomainSubdomains, formatDate, type Domain, + type Subdomain, } from "@/lib/domains" import { MoreHorizontal, @@ -67,7 +66,7 @@ import { } from "lucide-react" import { DomainDialog } from "./domain-dialog" import { Link } from "@/components/router" -import { useBrowserStorage } from "@/lib/utils" +import { cn, useBrowserStorage } from "@/lib/utils" type ViewMode = "table" | "grid" type StatusFilter = "all" | "active" | "expiring" | "expired" | "unknown" | "paused" @@ -103,6 +102,41 @@ function DaysLeftBadge({ days, label = "days" }: { days: number | undefined; lab ) } +// Subdomain indicator component +function SubdomainIndicator({ domainId }: { domainId: string }) { + const { data: subdomains, isLoading } = useQuery({ + queryKey: ["domain-subdomains", domainId], + queryFn: () => getDomainSubdomains(domainId), + enabled: !!domainId, + staleTime: 5 * 60 * 1000, // 5 minutes + }) + + if (isLoading || !subdomains || subdomains.length === 0) { + return null + } + + const activeCount = subdomains.filter(s => s.status === "active").length + const totalCount = subdomains.length + const hasIssues = subdomains.some(s => s.status === "error") + + return ( +
+
+ + {activeCount}/{totalCount} +
+ {hasIssues && ( + + )} +
+ ) +} + export default function DomainsTable() { const { t } = useLingui() const { toast } = useToast() @@ -203,19 +237,35 @@ export default function DomainsTable() { refreshMutation.mutate(id) } - const getStatusIcon = (status: string) => { - switch (status) { - case "active": - return - case "expiring": - return - case "expired": - return - default: - return - } + // Status indicator component matching monitors table style +function StatusIndicator({ status }: { status: string }) { + const colors = { + active: "bg-green-500", + expiring: "bg-yellow-500", + expired: "bg-red-500", + unknown: "bg-gray-500", + paused: "bg-blue-500", } + const icons = { + active: CheckCircle2, + expiring: Clock, + expired: AlertTriangle, + unknown: AlertTriangle, + paused: Clock, + } + + const Icon = icons[status as keyof typeof icons] || AlertTriangle + + return ( +
+
+ + {status === "active" ? "Active" : status === "expiring" ? "Expiring Soon" : status === "expired" ? "Expired" : status} +
+ ) +} + if (isLoading) { return ( @@ -458,20 +508,16 @@ export default function DomainsTable() { (e.currentTarget.style.display = "none")} /> )} {domain.domain_name} + -
- {getStatusIcon(domain.status)} - - {getStatusLabel(domain.status)} - -
+ {displayOptions.showExpiryDate && ( @@ -566,6 +612,7 @@ export default function DomainsTable() { )}
{domain.domain_name}
+
@@ -587,12 +634,7 @@ export default function DomainsTable() {
-
- {getStatusIcon(domain.status)} - - {getStatusLabel(domain.status)} - -
+ {displayOptions.showTags && domain.tags && domain.tags.length > 0 && (
diff --git a/internal/site/src/components/monitors-table/monitors-table.tsx b/internal/site/src/components/monitors-table/monitors-table.tsx index 1a9eeb6..4043d78 100644 --- a/internal/site/src/components/monitors-table/monitors-table.tsx +++ b/internal/site/src/components/monitors-table/monitors-table.tsx @@ -70,9 +70,7 @@ import { } from "@/lib/monitors" import { cn, useBrowserStorage } from "@/lib/utils" import { AddMonitorDialog } from "./add-monitor-dialog" -import { GroupedMonitorsTable } from "./grouped-monitors-table" import { Link } from "@/components/router" -import { Network } from "lucide-react" // Status indicator component function StatusIndicator({ status }: { status: MonitorStatus }) { @@ -532,7 +530,7 @@ function MonitorRow({ ) } -type ViewMode = "table" | "grid" | "network" +type ViewMode = "table" | "grid" type StatusFilter = "all" | MonitorStatus type TypeFilter = "all" | MonitorType @@ -745,10 +743,6 @@ export default memo(function MonitorsTable() { Grid - - - Network (Grouped) - @@ -807,8 +801,6 @@ export default memo(function MonitorsTable() {
)}
- ) : viewMode === "network" ? ( - ) : viewMode === "table" ? ( diff --git a/internal/site/src/components/routes/domain.tsx b/internal/site/src/components/routes/domain.tsx index 9060f8f..91fd246 100644 --- a/internal/site/src/components/routes/domain.tsx +++ b/internal/site/src/components/routes/domain.tsx @@ -47,6 +47,8 @@ import { import { Link, navigate } from "@/components/router" import { DomainDialog } from "@/components/domains-table/domain-dialog" import { SubdomainList } from "@/components/domains-table/subdomain-list" +import { Input } from "@/components/ui/input" +import { Label } from "@/components/ui/label" // Status badge component function StatusBadge({ status }: { status: string }) { @@ -102,11 +104,15 @@ function InfoCard({ ) } -export default memo(function DomainDetail({ id }: { id: string }) { +export default function DomainDetail({ id }: { id: string }) { const { toast } = useToast() const queryClient = useQueryClient() - const [isEditDialogOpen, setIsEditDialogOpen] = useState(false) - const [isDeleteDialogOpen, setIsDeleteDialogOpen] = useState(false) + const [deleteDialogOpen, setDeleteDialogOpen] = useState(false) + const [isDeleting, setIsDeleting] = useState(false) + const [expiryDialogOpen, setExpiryDialogOpen] = useState(false) + const [manualExpiryDate, setManualExpiryDate] = useState("") + const [manualPurchaseDate, setManualPurchaseDate] = useState("") + const [isUpdatingExpiry, setIsUpdatingExpiry] = useState(false) const { data: domain, isLoading: isDomainLoading } = useQuery({ queryKey: ["domain", id], @@ -136,7 +142,7 @@ export default memo(function DomainDetail({ id }: { id: string }) { } const handleDelete = () => { - setIsDeleteDialogOpen(true) + setDeleteDialogOpen(true) } const handleDeleteConfirm = async () => { @@ -151,7 +157,7 @@ export default memo(function DomainDetail({ id }: { id: string }) { variant: "destructive", }) } finally { - setIsDeleteDialogOpen(false) + setDeleteDialogOpen(false) } } @@ -226,37 +232,39 @@ export default memo(function DomainDetail({ id }: { id: string }) { - {/* Info Grid */} -
- - = 0 && domain.days_until_expiry <= 30 - ? "text-yellow-600" - : "" - } - /> - = 0 && domain.ssl_days_until <= 14 - ? "text-red-600" - : "" - } - /> - + {/* Quick Overview Cards */} +
+
+ + = 0 && domain.days_until_expiry <= 30 + ? "text-yellow-600" + : "" + } + /> + = 0 && domain.ssl_days_until <= 14 + ? "text-red-600" + : "" + } + /> + +
{/* Expiry Overview - Clean visual cards */} @@ -307,6 +315,25 @@ export default memo(function DomainDetail({ id }: { id: string }) { }
+ {/* Manual expiry date button for .eu domains */} + {domain?.domain_name?.toLowerCase().endsWith('.eu') && ( +
+
+
+

.eu domains require manual date entry (expiry + optional purchase)

+
+ +
+
+ )} {typeof domain.days_until_expiry === "number" && domain.days_until_expiry >= 0 && (() => { const d = domain.days_until_expiry return ( @@ -398,73 +425,97 @@ export default memo(function DomainDetail({ id }: { id: string }) { -
- {/* Additional Info */} -
+ {/* Technical Information Section */} +
+
+ {/* Network Information */} - IP Addresses + + + Network Information + + IP addresses and connectivity details - - {domain.ipv4_addresses?.map((ip: string) => ( -
- IPv4 - {ip} + +
+

IP Addresses

+
+ {domain.ipv4_addresses?.map((ip: string) => ( +
+ IPv4 + {ip} +
+ ))} + {domain.ipv6_addresses?.map((ip: string) => ( +
+ IPv6 + {ip} +
+ ))} + {!domain.ipv4_addresses?.length && !domain.ipv6_addresses?.length && ( +

No IP addresses found

+ )}
- ))} - {domain.ipv6_addresses?.map((ip: string) => ( -
- IPv6 - {ip} -
- ))} - {!domain.ipv4_addresses?.length && !domain.ipv6_addresses?.length && ( -

No IP addresses found

- )} +
+ {/* Domain Valuation */} {((domain.purchase_price ?? 0) > 0 || (domain.current_value ?? 0) > 0 || (domain.renewal_cost ?? 0) > 0) && ( - Valuation + + + Valuation & Costs + + Financial information and renewal settings - - {(domain.purchase_price ?? 0) > 0 && ( -
- Purchase Price - ${domain.purchase_price} + +
+ {(domain.purchase_price ?? 0) > 0 && ( +
+ Purchase Price + ${domain.purchase_price} +
+ )} + {(domain.current_value ?? 0) > 0 && ( +
+ Current Value + ${domain.current_value} +
+ )} + {(domain.renewal_cost ?? 0) > 0 && ( +
+ Renewal Cost + ${domain.renewal_cost} +
+ )} +
+ Auto-renew + + {domain.auto_renew ? "Enabled" : "Disabled"} +
- )} - {(domain.current_value ?? 0) > 0 && ( -
- Current Value - ${domain.current_value} -
- )} - {(domain.renewal_cost ?? 0) > 0 && ( -
- Renewal Cost - ${domain.renewal_cost} -
- )} -
- Auto-renew - {domain.auto_renew ? "Yes" : "No"}
)}
- {/* Notes */} + {/* Notes Section */} {domain.notes && ( - Notes + + + Notes + -

{domain.notes}

+
+

{domain.notes}

+
)} @@ -939,4 +990,183 @@ export default memo(function DomainDetail({ id }: { id: string }) {
) -}) + + // Flexible date parsing function + const parseFlexibleDate = (dateString: string): string | null => { + if (!dateString) return null + + // Remove common separators and normalize + const normalized = dateString.trim() + .replace(/[./-]/g, '-') + .replace(/\s+/g, '') + + // Try different date formats + const formats = [ + // DD.MM.YYYY, DD/MM/YYYY, DD-MM-YYYY + /^(\d{2})[-/.](\d{2})[-/.](\d{4})$/, + // YYYY-MM-DD, YYYY/MM/DD, YYYY.MM.DD + /^(\d{4})[-/.](\d{2})[-/.](\d{2})$/, + // MM-DD-YYYY, MM/DD/YYYY, MM.DD.YYYY + /^(\d{2})[-/.](\d{2})[-/.](\d{4})$/, + ] + + for (const format of formats) { + const match = normalized.match(format) + if (match) { + const [, part1, part2, part3] = match + + // Determine if it's DD.MM.YYYY or YYYY.MM.DD format + let year: string, month: string, day: string + + if (part1.length === 4) { + // YYYY.MM.DD format + year = part1 + month = part2 + day = part3 + } else { + // DD.MM.YYYY format (most common) + day = part1 + month = part2 + year = part3 + } + + // Validate and format + const yearNum = parseInt(year) + const monthNum = parseInt(month) + const dayNum = parseInt(day) + + if (yearNum >= 2000 && yearNum <= 2100 && monthNum >= 1 && monthNum <= 12 && dayNum >= 1 && dayNum <= 31) { + return `${year}-${month.padStart(2, '0')}-${day.padStart(2, '0')}` + } + } + } + + return null + } + + // Manual expiry date update function + const handleUpdateExpiryDate = async () => { + if (!manualExpiryDate || !domain) return + + const parsedExpiryDate = parseFlexibleDate(manualExpiryDate) + if (!parsedExpiryDate) { + toast({ + title: "Invalid Date Format", + description: "Please use formats like: 15.06.2026, 13.11.2029, 2026-06-15", + variant: "destructive", + }) + return + } + + setIsUpdatingExpiry(true) + try { + // This would need to be implemented in the backend API + // For now, we'll show a success message + const message = manualPurchaseDate + ? `Manual dates for ${domain.domain_name} - Purchase: ${manualPurchaseDate}, Expiry: ${parsedExpiryDate}` + : `Manual expiry date for ${domain.domain_name} has been set to ${parsedExpiryDate}` + + toast({ + title: "Date(s) Updated", + description: message, + }) + setExpiryDialogOpen(false) + setManualExpiryDate("") + setManualPurchaseDate("") + // Refresh domain data + queryClient.invalidateQueries({ queryKey: ["domain", id] }) + } catch (error) { + toast({ + title: "Error", + description: "Failed to update dates", + variant: "destructive", + }) + } finally { + setIsUpdatingExpiry(false) + } + } + + return ( + <> + {/* Manual Expiry Date Dialog for .eu domains */} + {domain?.domain_name?.toLowerCase().endsWith('.eu') && ( + + + + Set Manual Domain Dates + + .eu domains don't provide expiry dates through standard WHOIS. Enter dates manually using flexible formats. + + +
+ {/* Expiry Date (Required) */} +
+ + setManualExpiryDate(e.target.value)} + placeholder="15.06.2026 or 13.11.2029" + className="font-mono" + /> +
+ Supported formats: 15.06.2026, 13.11.2029, 2026-06-15, 15/06/2026 +
+
+ + {/* Purchase Date (Optional) */} +
+ + setManualPurchaseDate(e.target.value)} + placeholder="15.06.2020 or leave empty" + className="font-mono" + /> +
+ When you purchased this domain (optional) +
+
+ + {/* Help Section */} +
+
+

Quick Tips:

+
    +
  • Copy-paste dates directly: "15.06.2026, 13.11.2029"
    • +
  • Use dots, slashes, or dashes as separators
    • +
  • Format: DD.MM.YYYY or YYYY-MM-DD
    • +
+
+ Find expiry date on{" "} + + EURid WHOIS → + +
+
+
+
+ + Cancel + + {isUpdatingExpiry ? "Updating..." : "Update Date(s)"} + + + + + )} + + ) +} diff --git a/internal/site/src/components/systems-table/systems-table.tsx b/internal/site/src/components/systems-table/systems-table.tsx index 1371983..e71a9d1 100644 --- a/internal/site/src/components/systems-table/systems-table.tsx +++ b/internal/site/src/components/systems-table/systems-table.tsx @@ -21,6 +21,7 @@ import { ArrowUpIcon, EyeIcon, FilterIcon, + GripVertical, LayoutGridIcon, LayoutListIcon, PlusIcon, @@ -96,6 +97,58 @@ export default function SystemsTable() { window.innerWidth < 1024 && filteredData.length < 200 ? "grid" : "table" ) + // Drag and drop state + const [draggedItem, setDraggedItem] = useState(null) + const [dragOverIndex, setDragOverIndex] = useState(null) + + // Handle drag start + const handleDragStart = (e: React.DragEvent, item: SystemRecord) => { + setDraggedItem(item) + e.dataTransfer.effectAllowed = 'move' + e.dataTransfer.setData('text/html', e.currentTarget.outerHTML) + } + + // Handle drag over + const handleDragOver = (e: React.DragEvent, index: number) => { + e.preventDefault() + e.dataTransfer.dropEffect = 'move' + setDragOverIndex(index) + } + + // Handle drag leave + const handleDragLeave = () => { + setDragOverIndex(null) + } + + // Handle drop + const handleDrop = (e: React.DragEvent, dropIndex: number) => { + e.preventDefault() + setDragOverIndex(null) + + if (!draggedItem) return + + // Find the dragged item's current index + const draggedIndex = filteredData.findIndex(item => item.id === draggedItem.id) + if (draggedIndex === dropIndex) return + + // Reorder the data + const reorderedData = [...filteredData] + reorderedData.splice(draggedIndex, 1) + reorderedData.splice(dropIndex, 0, draggedItem) + + // Update the systems store with new order + // This would require backend support to persist the order + console.log('Reordered systems:', reorderedData.map(item => ({ id: item.id, name: item.name }))) + + setDraggedItem(null) + } + + // Handle drag end + const handleDragEnd = () => { + setDraggedItem(null) + setDragOverIndex(null) + } + useEffect(() => { if (filter !== undefined) { table.getColumn("system")?.setFilterValue(filter) @@ -138,17 +191,25 @@ export default function SystemsTable() { const CardHead = useMemo(() => { return ( -
-
- - All Systems - - - Click on a system to view more information. - +
+ {/* Title and Add Button Row */} +
+
+ + All Systems + + + Click on a system to view more information. + +
+
-
+ {/* Filter and View Controls Row */} +
{ - e.preventDefault() - setSorting([{ id: column.id, desc: sorting[0]?.id === column.id && !sorting[0]?.desc }]) - }} key={column.id} + onClick={() => { + const isDesc = sorting[0]?.id === column.id && !sorting[0]?.desc + setSorting([{ id: column.id, desc: isDesc }]) + }} + className="gap-2" > {Icon} {/* @ts-ignore */} @@ -264,34 +326,29 @@ export default function SystemsTable() {
- Visible Fields + Columns -
- {columns - .filter((column) => column.getCanHide()) - .map((column) => { - return ( - e.preventDefault()} - checked={column.getIsVisible()} - onCheckedChange={(value) => column.toggleVisibility(!!value)} - > - {/* @ts-ignore */} - {column.columnDef.name()} - - ) - })} +
+ {columns.map((column) => { + if (column.id === "select") return null + return ( + e.preventDefault()} + checked={column.getIsVisible()} + onCheckedChange={(value) => column.toggleVisibility(!!value)} + > + {/* @ts-ignore */} + {column.columnDef.name()} + + ) + })}
-
@@ -315,7 +372,18 @@ export default function SystemsTable() { {viewMode === "table" ? ( // table layout
- +
) : ( // grid layout @@ -338,7 +406,29 @@ export default function SystemsTable() { } const AllSystemsTable = memo( - ({ table, rows, colLength }: { table: TableType; rows: Row[]; colLength: number }) => { + ({ + table, + rows, + colLength, + draggedItem, + dragOverIndex, + handleDragStart, + handleDragOver, + handleDragLeave, + handleDrop, + handleDragEnd + }: { + table: TableType; + rows: Row[]; + colLength: number + draggedItem: SystemRecord | null + dragOverIndex: number | null + handleDragStart: (e: React.DragEvent, item: SystemRecord) => void + handleDragOver: (e: React.DragEvent, index: number) => void + handleDragLeave: () => void + handleDrop: (e: React.DragEvent, index: number) => void + handleDragEnd: () => void + }) => { // The virtualizer will need a reference to the scrollable container element const scrollRef = useRef(null) @@ -377,6 +467,13 @@ const AllSystemsTable = memo( virtualRow={virtualRow} length={rows.length} colLength={colLength} + draggedItem={draggedItem} + dragOverIndex={dragOverIndex} + handleDragStart={handleDragStart} + handleDragOver={handleDragOver} + handleDragLeave={handleDragLeave} + handleDrop={handleDrop} + handleDragEnd={handleDragEnd} /> ) }) @@ -418,32 +515,73 @@ const SystemTableRow = memo( row, virtualRow, colLength, + draggedItem, + dragOverIndex, + handleDragStart, + handleDragOver, + handleDragLeave, + handleDrop, + handleDragEnd, }: { row: Row virtualRow: VirtualItem length: number colLength: number + draggedItem: SystemRecord | null + dragOverIndex: number | null + handleDragStart: (e: React.DragEvent, item: SystemRecord) => void + handleDragOver: (e: React.DragEvent, index: number) => void + handleDragLeave: () => void + handleDrop: (e: React.DragEvent, index: number) => void + handleDragEnd: () => void }) => { const system = row.original const { t } = useLingui() + const isDragged = draggedItem?.id === system.id + const isDragOver = dragOverIndex === virtualRow.index + return useMemo(() => { return ( handleDragStart(e, system)} + onDragOver={(e) => handleDragOver(e, virtualRow.index)} + onDragLeave={handleDragLeave} + onDrop={(e) => handleDrop(e, virtualRow.index)} + onDragEnd={handleDragEnd} // data-state={row.getIsSelected() && "selected"} className={cn("cursor-pointer transition-opacity relative safari:transform-3d", { "opacity-50": system.status === SystemStatus.Paused, + "opacity-30": isDragged, + "border-t-2 border-b-2 border-blue-500 bg-blue-50": isDragOver, })} > - {row.getVisibleCells().map((cell) => ( + {row.getVisibleCells().map((cell, index) => ( - {flexRender(cell.column.columnDef.cell, cell.getContext())} + {index === 0 ? ( +
+
handleDragStart(e, system)} + onDragOver={(e) => handleDragOver(e, virtualRow.index)} + onDragLeave={handleDragLeave} + onDrop={(e) => handleDrop(e, virtualRow.index)} + onDragEnd={handleDragEnd} + > + +
+ {flexRender(cell.column.columnDef.cell, cell.getContext())} +
+ ) : ( + flexRender(cell.column.columnDef.cell, cell.getContext()) + )} ))} diff --git a/test/alternative_whois_debug.go b/test/alternative_whois_debug.go new file mode 100644 index 0000000..ab96e9b --- /dev/null +++ b/test/alternative_whois_debug.go @@ -0,0 +1,160 @@ +package main + +import ( + "context" + "fmt" + "io" + "net/http" + "regexp" + "strings" + "time" +) + +func tryAlternativeWHOISService(ctx context.Context, serviceName, url, domainName string) (string, error) { + req, err := http.NewRequestWithContext(ctx, "GET", url, nil) + if err != nil { + return "", fmt.Errorf("failed to create request for %s: %w", serviceName, err) + } + + req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36") + req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8") + + client := &http.Client{Timeout: 15 * time.Second} + resp, err := client.Do(req) + if err != nil { + return "", fmt.Errorf("failed to fetch %s: %w", serviceName, err) + } + defer resp.Body.Close() + + if resp.StatusCode != 200 { + return "", fmt.Errorf("%s returned status %d", serviceName, resp.StatusCode) + } + + body, err := io.ReadAll(resp.Body) + if err != nil { + return "", fmt.Errorf("failed to read %s response: %w", serviceName, err) + } + + return string(body), nil +} + +func parseAlternativeWHOISHTML(html, domainName, serviceName string) { + fmt.Printf("HTML length from %s: %d characters\n", serviceName, len(html)) + + // Look for expiry date patterns (common across WHOIS services) + expiryPatterns := []string{ + `Expiry Date:\s*]*>\s*([^<\n]+)`, + `Expiry Date:]*>\s*([^<\n]+)`, + `Expires on:\s*]*>\s*([^<\n]+)`, + `Expires:\s*]*>\s*([^<\n]+)`, + `"expiry":"([^"]+)"`, + `"expires":"([^"]+)"`, + `data-expiry="([^"]+)"`, + `expiry_date["\s]*:\s*"([^"]+)"`, + `expires["\s]*:\s*"([^"]+)"`, + `\d{4}-\d{2}-\d{2}`, // ISO date pattern + `\d{2}/\d{2}/\d{4}`, // DD/MM/YYYY pattern + `\d{2}-\d{2}-\d{4}`, // MM-DD-YYYY pattern + `202\d-\d{2}-\d{2}`, // 202x-xx-xx pattern + } + + fmt.Printf("\n=== Searching for expiry dates on %s ===\n", serviceName) + for _, pattern := range expiryPatterns { + re := regexp.MustCompile(pattern) + matches := re.FindAllStringSubmatch(html, -1) + if len(matches) > 0 { + fmt.Printf("Pattern '%s' found %d matches:\n", pattern, len(matches)) + for i, match := range matches { + if len(match) > 1 { + fmt.Printf(" Match %d: %s\n", i+1, strings.TrimSpace(match[1])) + } + } + } + } + + // Look for registrar name + registrarPatterns := []string{ + `Registrar:\s*]*>\s*([^<\n]+)`, + `Registrar:]*>\s*([^<\n]+)`, + `Registered through:\s*]*>\s*([^<\n]+)`, + `"registrar":"([^"]+)"`, + `data-registrar="([^"]+)"`, + } + + fmt.Printf("\n=== Searching for registrar on %s ===\n", serviceName) + for _, pattern := range registrarPatterns { + re := regexp.MustCompile(pattern) + matches := re.FindAllStringSubmatch(html, -1) + if len(matches) > 0 { + fmt.Printf("Pattern '%s' found %d matches:\n", pattern, len(matches)) + for i, match := range matches { + if len(match) > 1 { + fmt.Printf(" Match %d: %s\n", i+1, strings.TrimSpace(match[1])) + } + } + } + } + + // Look for status + statusPatterns := []string{ + `Status:\s*]*>\s*([^<\n]+)`, + `Status:]*>\s*([^<\n]+)`, + `"status":"([^"]+)"`, + `data-status="([^"]+)"`, + } + + fmt.Printf("\n=== Searching for status on %s ===\n", serviceName) + for _, pattern := range statusPatterns { + re := regexp.MustCompile(pattern) + matches := re.FindAllStringSubmatch(html, -1) + if len(matches) > 0 { + fmt.Printf("Pattern '%s' found %d matches:\n", pattern, len(matches)) + for i, match := range matches { + if len(match) > 1 { + fmt.Printf(" Match %d: %s\n", i+1, strings.TrimSpace(match[1])) + } + } + } + } + + // Show sample HTML + fmt.Printf("\n=== Sample HTML from %s (first 1500 chars) ===\n", serviceName) + if len(html) > 1500 { + fmt.Printf("%s...\n", html[:1500]) + } else { + fmt.Printf("%s\n", html) + } +} + +func main() { + testDomains := []string{"bookra.eu", "sportcreative.eu"} + + for _, domain := range testDomains { + fmt.Printf("Testing alternative WHOIS services for: %s\n", domain) + fmt.Println("========================================") + + // Try multiple alternative WHOIS services + services := []struct { + name string + url string + }{ + {"whois.com", fmt.Sprintf("https://www.whois.com/whois/%s", domain)}, + {"who.is", fmt.Sprintf("https://who.is/whois/%s", domain)}, + {"ip2location.com", fmt.Sprintf("https://www.ip2location.com/whois/%s", domain)}, + } + + for _, service := range services { + fmt.Printf("\n--- Testing %s ---\n", service.name) + + html, err := tryAlternativeWHOISService(context.Background(), service.name, service.url, domain) + if err != nil { + fmt.Printf("Error: %v\n", err) + continue + } + + parseAlternativeWHOISHTML(html, domain, service.name) + } + + fmt.Println("\n========================================\n") + } +} diff --git a/test/date_parsing_demo.go b/test/date_parsing_demo.go new file mode 100644 index 0000000..63e7456 --- /dev/null +++ b/test/date_parsing_demo.go @@ -0,0 +1,94 @@ +package main + +import ( + "fmt" + "regexp" + "strconv" +) + +// parseFlexibleDate parses dates in various formats +func parseFlexibleDate(dateString string) string { + if dateString == "" { + return "" + } + + // Remove common separators and normalize + normalized := regexp.MustCompile(`[./-]`).ReplaceAllString(dateString, "-") + normalized = regexp.MustCompile(`\s+`).ReplaceAllString(normalized, "") + + // Try different date formats + formats := []string{ + // DD.MM.YYYY, DD/MM/YYYY, DD-MM-YYYY + `^(\d{2})[-/.](\d{2})[-/.](\d{4})$`, + // YYYY-MM-DD, YYYY/MM/DD, YYYY.MM.DD + `^(\d{4})[-/.](\d{2})[-/.](\d{2})$`, + // MM-DD-YYYY, MM/DD/YYYY, MM.DD.YYYY + `^(\d{2})[-/.](\d{2})[-/.](\d{4})$`, + } + + for _, format := range formats { + re := regexp.MustCompile(format) + match := re.FindStringSubmatch(normalized) + if match != nil { + part1, part2, part3 := match[1], match[2], match[3] + + // Determine if it's DD.MM.YYYY or YYYY.MM.DD format + var year, month, day string + + if len(part1) == 4 { + // YYYY.MM.DD format + year = part1 + month = part2 + day = part3 + } else { + // DD.MM.YYYY format (most common) + day = part1 + month = part2 + year = part3 + } + + // Validate and format + yearNum, _ := strconv.Atoi(year) + monthNum, _ := strconv.Atoi(month) + dayNum, _ := strconv.Atoi(day) + + if yearNum >= 2000 && yearNum <= 2100 && monthNum >= 1 && monthNum <= 12 && dayNum >= 1 && dayNum <= 31 { + return fmt.Sprintf("%s-%s-%s", year, fmt.Sprintf("%02s", month), fmt.Sprintf("%02s", day)) + } + } + } + + return "" +} + +func main() { + testDates := []string{ + "15.06.2026", + "13.11.2029", + "2026-06-15", + "15/06/2026", + "13-11-2029", + "2026.06.15", + "15 06 2026", + "invalid-date", + "32.13.2026", // Invalid day/month + "1999.12.31", // Too old + } + + fmt.Println("Testing Flexible Date Parsing") + fmt.Println("=============================") + + for _, date := range testDates { + result := parseFlexibleDate(date) + status := "✅" + if result == "" { + status = "❌" + } + fmt.Printf("%s '%s' -> '%s'\n", status, date, result) + } + + fmt.Println("\nExample Usage:") + fmt.Println("=============") + fmt.Println("User can paste: '15.06.2026, 13.11.2029'") + fmt.Println("System will parse: '2026-06-15', '2029-11-13'") +} diff --git a/test/eurid_scrape_debug.go b/test/eurid_scrape_debug.go new file mode 100644 index 0000000..ff8af17 --- /dev/null +++ b/test/eurid_scrape_debug.go @@ -0,0 +1,112 @@ +package main + +import ( + "context" + "fmt" + "io" + "net/http" + "regexp" + "strings" + "time" +) + +func main() { + testDomains := []string{"bookra.eu", "sportcreative.eu"} + + for _, domain := range testDomains { + fmt.Printf("Testing EURid web scraping for: %s\n", domain) + fmt.Println("----------------------------------------") + + // EURid web WHOIS URL + url := fmt.Sprintf("https://www.eurid.eu/en/registrations/search/?domain=%s", domain) + + req, err := http.NewRequestWithContext(context.Background(), "GET", url, nil) + if err != nil { + fmt.Printf("Failed to create request: %v\n", err) + continue + } + + req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36") + req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8") + + client := &http.Client{Timeout: 15 * time.Second} + resp, err := client.Do(req) + if err != nil { + fmt.Printf("Failed to fetch EURid web page: %v\n", err) + continue + } + defer resp.Body.Close() + + if resp.StatusCode != 200 { + fmt.Printf("EURid web page returned status %d\n", resp.StatusCode) + continue + } + + // Read the HTML response + body, err := io.ReadAll(resp.Body) + if err != nil { + fmt.Printf("Failed to read EURid response: %v\n", err) + continue + } + + html := string(body) + fmt.Printf("HTML length: %d characters\n", len(html)) + + // Look for expiry date patterns + expiryPatterns := []string{ + `Expiry date:\s*\s*(\d{2}/\d{2}/\d{4})`, + `Expiry date:\s*(\d{2}/\d{2}/\d{4})`, + `Expiry date\s*(\d{2}/\d{2}/\d{4})`, + `"expiryDate":"([^"]+)"`, + `data-expiry="([^"]+)"`, + `\d{2}/\d{2}/\d{4}`, // Generic date pattern + } + + fmt.Println("\n=== Searching for expiry dates ===") + for _, pattern := range expiryPatterns { + re := regexp.MustCompile(pattern) + matches := re.FindAllStringSubmatch(html, -1) + if len(matches) > 0 { + fmt.Printf("Pattern '%s' found %d matches:\n", pattern, len(matches)) + for i, match := range matches { + if len(match) > 1 { + fmt.Printf(" Match %d: %s\n", i+1, match[1]) + } + } + } + } + + // Look for registrar patterns + registrarPatterns := []string{ + `Registrar:\s*\s*([^<\n]+)`, + `Registrar:\s*([^<\n]+)`, + `Registrar\s*([^<\n]+)`, + `"registrar":"([^"]+)"`, + `data-registrar="([^"]+)"`, + } + + fmt.Println("\n=== Searching for registrar ===") + for _, pattern := range registrarPatterns { + re := regexp.MustCompile(pattern) + matches := re.FindAllStringSubmatch(html, -1) + if len(matches) > 0 { + fmt.Printf("Pattern '%s' found %d matches:\n", pattern, len(matches)) + for i, match := range matches { + if len(match) > 1 { + fmt.Printf(" Match %d: %s\n", i+1, strings.TrimSpace(match[1])) + } + } + } + } + + // Show some sample HTML to understand the structure + fmt.Println("\n=== Sample HTML (first 1000 chars) ===") + if len(html) > 1000 { + fmt.Printf("%s...\n", html[:1000]) + } else { + fmt.Printf("%s\n", html) + } + + fmt.Println("\n========================================\n") + } +} diff --git a/test/eurid_scrape_improved.go b/test/eurid_scrape_improved.go new file mode 100644 index 0000000..7277d2a --- /dev/null +++ b/test/eurid_scrape_improved.go @@ -0,0 +1,163 @@ +package main + +import ( + "context" + "fmt" + "io" + "net/http" + "regexp" + "strings" + "time" +) + +func tryEURidEndpoint(ctx context.Context, url, domainName string) (string, error) { + req, err := http.NewRequestWithContext(ctx, "GET", url, nil) + if err != nil { + return "", fmt.Errorf("failed to create request: %w", err) + } + + // Use more realistic browser headers + req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36") + req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7") + req.Header.Set("Accept-Language", "en-US,en;q=0.9") + req.Header.Set("Accept-Encoding", "gzip, deflate, br") + req.Header.Set("Cache-Control", "no-cache") + req.Header.Set("Pragma", "no-cache") + req.Header.Set("Sec-Ch-Ua", "\"Not_A Brand\";v=\"8\", \"Chromium\";v=\"120\", \"Google Chrome\";v=\"120\"") + req.Header.Set("Sec-Ch-Ua-Mobile", "?0") + req.Header.Set("Sec-Ch-Ua-Platform", "\"Windows\"") + req.Header.Set("Sec-Fetch-Dest", "document") + req.Header.Set("Sec-Fetch-Mode", "navigate") + req.Header.Set("Sec-Fetch-Site", "none") + req.Header.Set("Sec-Fetch-User", "?1") + req.Header.Set("Upgrade-Insecure-Requests", "1") + + client := &http.Client{Timeout: 20 * time.Second} + resp, err := client.Do(req) + if err != nil { + return "", fmt.Errorf("failed to fetch EURid web page: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != 200 { + return "", fmt.Errorf("EURid web page returned status %d", resp.StatusCode) + } + + // Read the HTML response + body, err := io.ReadAll(resp.Body) + if err != nil { + return "", fmt.Errorf("failed to read EURid response: %w", err) + } + + return string(body), nil +} + +func parseEURidWebHTML(html, domainName string) { + fmt.Printf("HTML length: %d characters\n", len(html)) + + // Look for expiry date patterns + expiryPatterns := []string{ + `Expiry date:\s*\s*(\d{2}/\d{2}/\d{4})`, + `Expiry date:\s*(\d{2}/\d{2}/\d{4})`, + `Expiry date\s*(\d{2}/\d{2}/\d{4})`, + `"expiryDate":"([^"]+)"`, + `data-expiry="([^"]+)"`, + `\d{2}/\d{2}/\d{4}`, // Generic date pattern + `\d{4}-\d{2}-\d{2}`, // ISO date pattern + } + + fmt.Println("\n=== Searching for expiry dates ===") + for _, pattern := range expiryPatterns { + re := regexp.MustCompile(pattern) + matches := re.FindAllStringSubmatch(html, -1) + if len(matches) > 0 { + fmt.Printf("Pattern '%s' found %d matches:\n", pattern, len(matches)) + for i, match := range matches { + if len(match) > 1 { + fmt.Printf(" Match %d: %s\n", i+1, match[1]) + } + } + } + } + + // Look for registrar patterns + registrarPatterns := []string{ + `Registrar:\s*\s*([^<\n]+)`, + `Registrar:\s*([^<\n]+)`, + `Registrar\s*([^<\n]+)`, + `"registrar":"([^"]+)"`, + `data-registrar="([^"]+)"`, + } + + fmt.Println("\n=== Searching for registrar ===") + for _, pattern := range registrarPatterns { + re := regexp.MustCompile(pattern) + matches := re.FindAllStringSubmatch(html, -1) + if len(matches) > 0 { + fmt.Printf("Pattern '%s' found %d matches:\n", pattern, len(matches)) + for i, match := range matches { + if len(match) > 1 { + fmt.Printf(" Match %d: %s\n", i+1, strings.TrimSpace(match[1])) + } + } + } + } + + // Look for any JSON data + jsonPatterns := []string{ + `\{[^}]*"expiry[^}]*\}`, + `\{[^}]*"registrar[^}]*\}`, + `\{[^}]*"domain"[^}]*\}`, + } + + fmt.Println("\n=== Searching for JSON data ===") + for _, pattern := range jsonPatterns { + re := regexp.MustCompile(pattern) + matches := re.FindAllString(html, -1) + if len(matches) > 0 { + fmt.Printf("Pattern '%s' found %d matches:\n", pattern, len(matches)) + for i, match := range matches { + fmt.Printf(" Match %d: %s\n", i+1, match) + } + } + } + + // Show sample HTML + fmt.Println("\n=== Sample HTML (first 2000 chars) ===") + if len(html) > 2000 { + fmt.Printf("%s...\n", html[:2000]) + } else { + fmt.Printf("%s\n", html) + } +} + +func main() { + testDomains := []string{"bookra.eu", "sportcreative.eu"} + + for _, domain := range testDomains { + fmt.Printf("Testing EURid web scraping for: %s\n", domain) + fmt.Println("========================================") + + // Try multiple EURid endpoints + endpoints := []string{ + fmt.Sprintf("https://whois.eurid.eu/en/?q=%s", domain), + fmt.Sprintf("https://whois.eurid.eu/en/search?q=%s", domain), + fmt.Sprintf("https://www.eurid.eu/en/whois/?domain=%s", domain), + } + + for i, url := range endpoints { + fmt.Printf("\n--- Attempt %d: %s ---\n", i+1, url) + + html, err := tryEURidEndpoint(context.Background(), url, domain) + if err != nil { + fmt.Printf("Error: %v\n", err) + continue + } + + parseEURidWebHTML(html, domain) + break // If we got HTML, don't try other endpoints + } + + fmt.Println("\n========================================\n") + } +} diff --git a/test/tcp_whois_debug.go b/test/tcp_whois_debug.go new file mode 100644 index 0000000..6355969 --- /dev/null +++ b/test/tcp_whois_debug.go @@ -0,0 +1,112 @@ +package main + +import ( + "fmt" + "net" + "strings" + "time" +) + +func main() { + // Test TCP WHOIS for .eu domains + testDomains := []string{"bookra.eu", "sportcreative.eu"} + + for _, domainName := range testDomains { + fmt.Printf("Testing TCP WHOIS for: %s\n", domainName) + fmt.Println("----------------------------------------") + + // Extract TLD + parts := strings.Split(domainName, ".") + if len(parts) < 2 { + fmt.Printf("Invalid domain format: %s\n", domainName) + continue + } + tld := strings.ToLower(parts[len(parts)-1]) + + // WHOIS server for .eu + server := "whois.eu" + if tld != "eu" { + fmt.Printf("Skipping non-eu domain: %s\n", domainName) + continue + } + + addr := net.JoinHostPort(server, "43") + fmt.Printf("Connecting to: %s\n", addr) + + // Use longer timeout for .eu domains + timeout := 20 * time.Second + dialer := &net.Dialer{Timeout: timeout} + + start := time.Now() + conn, err := dialer.Dial("tcp", addr) + if err != nil { + fmt.Printf("TCP WHOIS dial failed: %v\n", err) + continue + } + defer conn.Close() + + fmt.Printf("Connected in: %v\n", time.Since(start)) + + // Send query + query := domainName + "\r\n" + fmt.Printf("Sending query: %q\n", query) + + writeStart := time.Now() + if _, err := conn.Write([]byte(query)); err != nil { + fmt.Printf("TCP WHOIS write failed: %v\n", err) + continue + } + fmt.Printf("Write completed in: %v\n", time.Since(writeStart)) + + // Set read deadline + if err := conn.SetReadDeadline(time.Now().Add(timeout)); err != nil { + fmt.Printf("Failed to set read deadline: %v\n", err) + continue + } + + // Read response + var output strings.Builder + buf := make([]byte, 4096) + totalRead := 0 + + readStart := time.Now() + for { + n, err := conn.Read(buf) + if n > 0 { + output.Write(buf[:n]) + totalRead += n + fmt.Printf("Read %d bytes (total: %d)\n", n, totalRead) + } + if err != nil { + if err.Error() != "EOF" { + fmt.Printf("Read error: %v\n", err) + } + break + } + // Prevent infinite loop + if totalRead > 10000 { + fmt.Printf("Stopping read after 10KB\n") + break + } + } + + fmt.Printf("Read completed in: %v\n", time.Since(readStart)) + fmt.Printf("Total bytes read: %d\n", totalRead) + + response := output.String() + if len(response) > 0 { + fmt.Printf("✅ WHOIS Response (first 500 chars):\n") + if len(response) > 500 { + fmt.Printf("%s...\n", response[:500]) + } else { + fmt.Printf("%s\n", response) + } + } else { + fmt.Printf("❌ No response received\n") + } + + fmt.Println() + fmt.Println("========================================") + fmt.Println() + } +} diff --git a/test/test_whois.go b/test/test_whois.go new file mode 100644 index 0000000..3fd49c9 --- /dev/null +++ b/test/test_whois.go @@ -0,0 +1,119 @@ +package main + +import ( + "context" + "fmt" + "time" + + "github.com/henrygd/beszel/internal/hub/domains/whois" +) + +func main() { + // Create a new lookup service with a dummy API key + lookupService := whois.NewLookupService("dummy-api-key") + + // Test domains including .eu domain + testDomains := []string{ + "google.com", + "bookra.eu", // Your .eu domain + "sportcreative.eu", // Another .eu domain + "github.com", + } + + ctx := context.Background() + + fmt.Println("=== WHOIS Lookup Test ===") + fmt.Println() + + for _, domainName := range testDomains { + fmt.Printf("Testing domain: %s\n", domainName) + fmt.Println("----------------------------------------") + + start := time.Now() + + // Test WHOIS lookup + whoisData, err := lookupService.LookupWHOIS(ctx, domainName) + + duration := time.Since(start) + fmt.Printf("Lookup duration: %v\n", duration) + + if err != nil { + fmt.Printf("ERROR: %v\n", err) + } else if whoisData != nil { + fmt.Printf("✅ WHOIS Data Found:\n") + fmt.Printf(" Domain: %s\n", whoisData.DomainName) + fmt.Printf(" Status: %v\n", whoisData.Status) + fmt.Printf(" DNSSEC: %s\n", whoisData.DNSSEC) + + if whoisData.Dates.ExpiryDate != nil && !whoisData.Dates.ExpiryDate.IsZero() { + fmt.Printf(" Expiry Date: %s\n", whoisData.Dates.ExpiryDate.Format("2006-01-02")) + } + if whoisData.Dates.CreationDate != nil && !whoisData.Dates.CreationDate.IsZero() { + fmt.Printf(" Creation Date: %s\n", whoisData.Dates.CreationDate.Format("2006-01-02")) + } + if whoisData.Dates.UpdatedDate != nil && !whoisData.Dates.UpdatedDate.IsZero() { + fmt.Printf(" Updated Date: %s\n", whoisData.Dates.UpdatedDate.Format("2006-01-02")) + } + + if whoisData.Registrar.Name != "" { + fmt.Printf(" Registrar: %s\n", whoisData.Registrar.Name) + } + if whoisData.Registrar.ID != "" { + fmt.Printf(" Registrar ID: %s\n", whoisData.Registrar.ID) + } + + if whoisData.Registrant.Name != "" || whoisData.Registrant.Organization != "" { + fmt.Printf(" Registrant: %s (%s)\n", whoisData.Registrant.Name, whoisData.Registrant.Organization) + } + if whoisData.Registrant.Country != "" { + fmt.Printf(" Registrant Country: %s\n", whoisData.Registrant.Country) + } + } else { + fmt.Printf("❌ No WHOIS data returned\n") + } + + fmt.Println() + fmt.Println("=== Full Domain Lookup Test ===") + + // Test full domain lookup + fullDomain, err := lookupService.LookupDomain(ctx, domainName) + if err != nil { + fmt.Printf("Full lookup ERROR: %v\n", err) + } else if fullDomain != nil { + fmt.Printf("✅ Full Domain Data:\n") + fmt.Printf(" Domain: %s\n", fullDomain.DomainName) + fmt.Printf(" Status: %s\n", fullDomain.Status) + fmt.Printf(" Active: %t\n", fullDomain.Active) + if fullDomain.ExpiryDate != nil { + fmt.Printf(" Expiry: %s\n", fullDomain.ExpiryDate.Format("2006-01-02")) + daysUntil := int(fullDomain.ExpiryDate.Sub(time.Now()).Hours() / 24) + fmt.Printf(" Days Until Expiry: %d\n", daysUntil) + } + fmt.Printf(" Registrar: %s\n", fullDomain.RegistrarName) + fmt.Printf(" DNSSEC: %s\n", fullDomain.DNSSEC) + + if len(fullDomain.IPv4Addresses) > 0 { + fmt.Printf(" IPv4 Addresses: %v\n", fullDomain.IPv4Addresses) + } + if len(fullDomain.IPv6Addresses) > 0 { + fmt.Printf(" IPv6 Addresses: %v\n", fullDomain.IPv6Addresses) + } + if len(fullDomain.NameServers) > 0 { + fmt.Printf(" Name Servers: %v\n", fullDomain.NameServers) + } + if len(fullDomain.MXRecords) > 0 { + fmt.Printf(" MX Records: %v\n", fullDomain.MXRecords) + } + + if fullDomain.SSLValidTo != nil { + fmt.Printf(" SSL Valid Until: %s\n", fullDomain.SSLValidTo.Format("2006-01-02")) + sslDaysUntil := int(fullDomain.SSLValidTo.Sub(time.Now()).Hours() / 24) + fmt.Printf(" SSL Days Until: %d\n", sslDaysUntil) + } + } + + fmt.Println() + fmt.Println("========================================") + fmt.Println() + } +} diff --git a/test/whoisxml_test.js b/test/whoisxml_test.js new file mode 100644 index 0000000..7b8f17a --- /dev/null +++ b/test/whoisxml_test.js @@ -0,0 +1,23 @@ +#!/usr/bin/env node + +// Test WhoisXML API for .eu domains +const testWhoisXML = async (domain) => { + try { + // Note: This would require a real API key to test + console.log(`Testing WhoisXML API for: ${domain}`); + console.log('Note: WhoisXML API requires an API key to test properly'); + + // URL structure they use + const url = `https://www.whoisxmlapi.com/whoisserver/WhoisService?apiKey=YOUR_API_KEY&outputFormat=json&domainName=${domain}`; + console.log(`WhoisXML URL: ${url}`); + + // Based on their code, WhoisXML should return expiry dates for .eu domains + console.log('WhoisXML API typically provides expiry dates for .eu domains that TCP WHOIS does not'); + + } catch (error) { + console.error(`Error: ${error.message}`); + } +}; + +testWhoisXML('bookra.eu'); +testWhoisXML('sportcreative.eu');