feat(hub,site): enhance domain intelligence and monitor performance
Build Docker images / Hub (push) Failing after 1m35s

Implement comprehensive domain data collection including provider detection (DNS, hosting, email, CA), HTTP headers, TLS certificate chains, and SEO metadata. Added PageSpeed Insights integration for monitors to track Core Web Vitals.

- **hub**:
  - Add provider detection logic for DNS, email, and hosting.
  - Expand `Domain` entity to include SEO, headers, certificates, and enhanced registration details.
  - Implement automated collection of TLD, WHOIS raw data, and host country codes.
  - Update scheduler to track changes in providers and security settings (privacy/transfer lock).
  - Add PageSpeed check endpoint to monitor API.
- **site**:
  - Update domain table and detail views to display new intelligence (providers, headers, SEO).
  - Implement PageSpeed metrics visualization with Core Web Vitals status indicators.
  - Add display options for provider information in the domain list.
- **db**:
  - Add migration for new domain collection fields.
This commit is contained in:
Tomas Dvorak
2026-05-14 13:33:03 +02:00
parent 0dd7db8a82
commit fe5c7eaa95
16 changed files with 1712 additions and 146 deletions
+34
View File
@@ -635,6 +635,19 @@ func (h *APIHandler) recordToResponse(record *core.Record) map[string]interface{
"registrant_state": record.GetString("registrant_state"),
"abuse_email": record.GetString("abuse_email"),
"abuse_phone": record.GetString("abuse_phone"),
"dns_provider": record.GetString("dns_provider"),
"hosting_provider": record.GetString("hosting_provider"),
"email_provider": record.GetString("email_provider"),
"ca_provider": record.GetString("ca_provider"),
"headers": record.Get("headers"),
"certificates": record.Get("certificates"),
"seo_meta": record.Get("seo_meta"),
"whois_raw": record.GetString("whois_raw"),
"privacy_enabled": record.GetBool("privacy_enabled"),
"transfer_lock": record.GetBool("transfer_lock"),
"tld": record.GetString("tld"),
"domain_statuses": record.Get("domain_statuses"),
"host_country_code": record.GetString("host_country_code"),
"tags": record.Get("tags"),
"notes": record.GetString("notes"),
"favicon_url": record.GetString("favicon_url"),
@@ -717,6 +730,27 @@ func (h *APIHandler) applyLookupData(record *core.Record, domainData *domain.Dom
record.Set("registrant_postal", domainData.RegistrantPostal)
record.Set("abuse_email", domainData.AbuseEmail)
record.Set("abuse_phone", domainData.AbusePhone)
record.Set("dns_provider", domainData.DNSProvider)
record.Set("hosting_provider", domainData.HostingProvider)
record.Set("email_provider", domainData.EmailProvider)
record.Set("ca_provider", domainData.CAProvider)
if len(domainData.Headers) > 0 {
record.Set("headers", domainData.Headers)
}
if len(domainData.Certificates) > 0 {
record.Set("certificates", domainData.Certificates)
}
if domainData.SEOMeta != nil {
record.Set("seo_meta", domainData.SEOMeta)
}
record.Set("whois_raw", domainData.WHOISRaw)
record.Set("privacy_enabled", domainData.PrivacyEnabled)
record.Set("transfer_lock", domainData.TransferLock)
record.Set("tld", domainData.TLD)
if len(domainData.DomainStatuses) > 0 {
record.Set("domain_statuses", domainData.DomainStatuses)
}
record.Set("host_country_code", domainData.HostCountryCode)
record.Set("favicon_url", domainData.FaviconURL)
record.Set("last_checked", time.Now())
}
+275
View File
@@ -0,0 +1,275 @@
package detect
import (
"net/http"
"strings"
)
// ProviderInfo holds detected provider name
type ProviderInfo struct {
Name string
}
// DetectDNSProvider detects the DNS provider from NS records
func DetectDNSProvider(nsRecords []string) string {
for _, ns := range nsRecords {
nsLower := strings.ToLower(ns)
switch {
case strings.Contains(nsLower, "cloudflare"):
return "Cloudflare"
case strings.Contains(nsLower, "awsdns"):
return "Amazon Route 53"
case strings.Contains(nsLower, "googledomains") || strings.Contains(nsLower, "google.com"):
return "Google Domains"
case strings.Contains(nsLower, "namecheap") || strings.Contains(nsLower, "namecheaphosting"):
return "Namecheap"
case strings.Contains(nsLower, "godaddy"):
return "GoDaddy"
case strings.Contains(nsLower, "domaincontrol"):
return "GoDaddy"
case strings.Contains(nsLower, "nsone.net"):
return "NS1"
case strings.Contains(nsLower, "digitalocean"):
return "DigitalOcean"
case strings.Contains(nsLower, "linode"):
return "Linode"
case strings.Contains(nsLower, "vultr"):
return "Vultr"
case strings.Contains(nsLower, "he.net"):
return "Hurricane Electric"
case strings.Contains(nsLower, "dyn.com") || strings.Contains(nsLower, "dynect"):
return "Dyn (Oracle)"
case strings.Contains(nsLower, "ultradns"):
return "UltraDNS"
case strings.Contains(nsLower, "dnsimple"):
return "DNSimple"
case strings.Contains(nsLower, "hover"):
return "Hover"
case strings.Contains(nsLower, "register.com"):
return "Register.com"
case strings.Contains(nsLower, "enom"):
return "eNom"
case strings.Contains(nsLower, "worldnic"):
return "Network Solutions"
case strings.Contains(nsLower, "zoneedit"):
return "ZoneEdit"
case strings.Contains(nsLower, "easydns"):
return "EasyDNS"
case strings.Contains(nsLower, "gandi"):
return "Gandi"
case strings.Contains(nsLower, "ovh"):
return "OVH"
case strings.Contains(nsLower, "hetzner"):
return "Hetzner"
case strings.Contains(nsLower, "azure-dns"):
return "Microsoft Azure"
}
}
return ""
}
// DetectEmailProvider detects the email provider from MX records
func DetectEmailProvider(mxRecords []string) string {
for _, mx := range mxRecords {
mxLower := strings.ToLower(mx)
// Extract just the hostname part if it has priority prefix
host := mxLower
if idx := strings.Index(mxLower, " "); idx > 0 {
host = strings.TrimSpace(mxLower[idx+1:])
}
host = strings.TrimSuffix(host, ".")
switch {
case strings.Contains(host, "google") || strings.Contains(host, "gmail"):
return "Google Workspace"
case strings.Contains(host, "outlook") || strings.Contains(host, "microsoft") || strings.Contains(host, "protection.outlook"):
return "Microsoft 365"
case strings.Contains(host, "purelymail"):
return "Purelymail"
case strings.Contains(host, "zoho"):
return "Zoho Mail"
case strings.Contains(host, "protonmail") || strings.Contains(host, "pm.me"):
return "ProtonMail"
case strings.Contains(host, "fastmail"):
return "Fastmail"
case strings.Contains(host, "tutanota"):
return "Tutanota"
case strings.Contains(host, "mxroute"):
return "MXroute"
case strings.Contains(host, "namecheap"):
return "Namecheap"
case strings.Contains(host, "icloud") || strings.Contains(host, "me.com"):
return "iCloud Mail"
case strings.Contains(host, "yahoo"):
return "Yahoo"
case strings.Contains(host, "qq.com"):
return "QQ Mail"
case strings.Contains(host, "mail.ru"):
return "Mail.ru"
case strings.Contains(host, "yandex"):
return "Yandex"
case strings.Contains(host, "hover"):
return "Hover"
case strings.Contains(host, "godaddy") || strings.Contains(host, "domaincontrol"):
return "GoDaddy"
case strings.Contains(host, "pobox"):
return "Pobox"
case strings.Contains(host, "runbox"):
return "Runbox"
case strings.Contains(host, "posteo"):
return "Posteo"
case strings.Contains(host, "mailbox.org"):
return "Mailbox.org"
case strings.Contains(host, "forwardemail"):
return "Forward Email"
case strings.Contains(host, "improvmx"):
return "ImprovMX"
case strings.Contains(host, "cloudflare"):
return "Cloudflare Email Routing"
case strings.Contains(host, "amazonaws") || strings.Contains(host, "aws"):
return "Amazon SES"
case strings.Contains(host, "sendgrid") || strings.Contains(host, "twilio"):
return "SendGrid"
case strings.Contains(host, "mailgun"):
return "Mailgun"
case strings.Contains(host, "postmark"):
return "Postmark"
}
}
return ""
}
// DetectHostingProvider detects the hosting provider from HTTP headers
func DetectHostingProvider(headers http.Header) string {
server := strings.ToLower(headers.Get("Server"))
poweredBy := strings.ToLower(headers.Get("X-Powered-By"))
cfRay := headers.Get("CF-Ray")
vercelCache := headers.Get("X-Vercel-Cache")
vercelID := headers.Get("X-Vercel-Id")
netlifyID := headers.Get("X-NF-Request-Id")
githubRequest := headers.Get("X-GitHub-Request-Id")
switch {
case vercelCache != "" || vercelID != "":
return "Vercel"
case netlifyID != "":
return "Netlify"
case githubRequest != "":
return "GitHub Pages"
case cfRay != "":
return "Cloudflare"
case strings.Contains(server, "cloudflare"):
return "Cloudflare"
case strings.Contains(server, "nginx") && vercelCache != "":
return "Vercel"
case strings.Contains(server, "awselb") || strings.Contains(server, "elb"):
return "AWS"
case strings.Contains(server, "amazon"):
return "AWS"
case strings.Contains(server, "microsoft-iis"):
return "Microsoft Azure"
case strings.Contains(server, "google") || strings.Contains(server, "gws"):
return "Google Cloud"
case strings.Contains(server, "heroku"):
return "Heroku"
case strings.Contains(server, "digitalocean"):
return "DigitalOcean"
case strings.Contains(server, "linode"):
return "Linode"
case strings.Contains(server, "ovh"):
return "OVH"
case strings.Contains(server, "hetzner"):
return "Hetzner"
case strings.Contains(server, "fastly"):
return "Fastly"
case strings.Contains(server, "bunnycdn"):
return "BunnyCDN"
case strings.Contains(server, "keycdn"):
return "KeyCDN"
case strings.Contains(server, "stackpath"):
return "StackPath"
case strings.Contains(server, "sucuri"):
return "Sucuri"
case strings.Contains(poweredBy, "next.js") || strings.Contains(poweredBy, "nextjs"):
return "Vercel"
case strings.Contains(poweredBy, "php"):
return "PHP"
case strings.Contains(server, "apache"):
return "Apache"
case strings.Contains(server, "nginx"):
return "nginx"
case strings.Contains(server, "caddy"):
return "Caddy"
case strings.Contains(server, "lighttpd"):
return "Lighttpd"
case strings.Contains(server, "litespeed"):
return "LiteSpeed"
case strings.Contains(server, "openresty"):
return "OpenResty"
case strings.Contains(server, "jetty"):
return "Jetty"
case strings.Contains(server, "tomcat"):
return "Tomcat"
case strings.Contains(server, "iis"):
return "IIS"
}
return ""
}
// DetectCertificateAuthority detects the CA from an issuer string
func DetectCertificateAuthority(issuer string) string {
issuerLower := strings.ToLower(issuer)
switch {
case strings.Contains(issuerLower, "let's encrypt"):
return "Let's Encrypt"
case strings.Contains(issuerLower, "digicert"):
return "DigiCert"
case strings.Contains(issuerLower, "sectigo") || strings.Contains(issuerLower, "comodoca"):
return "Sectigo"
case strings.Contains(issuerLower, "globalsign"):
return "GlobalSign"
case strings.Contains(issuerLower, "geotrust"):
return "GeoTrust"
case strings.Contains(issuerLower, "thawte"):
return "Thawte"
case strings.Contains(issuerLower, "rapidssl"):
return "RapidSSL"
case strings.Contains(issuerLower, "symantec"):
return "Symantec"
case strings.Contains(issuerLower, "entrust"):
return "Entrust"
case strings.Contains(issuerLower, "certum"):
return "Certum"
case strings.Contains(issuerLower, "go daddy") || strings.Contains(issuerLower, "godaddy"):
return "GoDaddy"
case strings.Contains(issuerLower, "amazon"):
return "Amazon"
case strings.Contains(issuerLower, "google") && strings.Contains(issuerLower, "trust"):
return "Google Trust Services"
case strings.Contains(issuerLower, "cloudflare"):
return "Cloudflare"
case strings.Contains(issuerLower, "zero ssl") || strings.Contains(issuerLower, "zerossl"):
return "ZeroSSL"
case strings.Contains(issuerLower, "ssl.com"):
return "SSL.com"
case strings.Contains(issuerLower, "buypass"):
return "Buypass"
case strings.Contains(issuerLower, "harica"):
return "HARICA"
case strings.Contains(issuerLower, " Actalis "):
return "Actalis"
case strings.Contains(issuerLower, "swisssign"):
return "SwissSign"
case strings.Contains(issuerLower, "telekom"):
return "Telekom"
case strings.Contains(issuerLower, "trustwave"):
return "Trustwave"
case strings.Contains(issuerLower, "identrust"):
return "IdenTrust"
case strings.Contains(issuerLower, "usertrust"):
return "UserTrust"
case strings.Contains(issuerLower, "isrg") || strings.Contains(issuerLower, "internet security research"):
return "Let's Encrypt"
}
return issuer
}
+76 -2
View File
@@ -262,9 +262,28 @@ func (s *Scheduler) checkDomain(record *core.Record) error {
if newData.AbuseEmail != "" {
record.Set("abuse_email", newData.AbuseEmail)
}
if newData.AbusePhone != "" {
record.Set("abuse_phone", newData.AbusePhone)
record.Set("abuse_phone", newData.AbusePhone)
record.Set("dns_provider", newData.DNSProvider)
record.Set("hosting_provider", newData.HostingProvider)
record.Set("email_provider", newData.EmailProvider)
record.Set("ca_provider", newData.CAProvider)
if len(newData.Headers) > 0 {
record.Set("headers", newData.Headers)
}
if len(newData.Certificates) > 0 {
record.Set("certificates", newData.Certificates)
}
if newData.SEOMeta != nil {
record.Set("seo_meta", newData.SEOMeta)
}
record.Set("whois_raw", newData.WHOISRaw)
record.Set("privacy_enabled", newData.PrivacyEnabled)
record.Set("transfer_lock", newData.TransferLock)
record.Set("tld", newData.TLD)
if len(newData.DomainStatuses) > 0 {
record.Set("domain_statuses", newData.DomainStatuses)
}
record.Set("host_country_code", newData.HostCountryCode)
record.Set("last_checked", time.Now())
// Update status - fallback to existing record expiry if new lookup didn't return one
@@ -422,6 +441,61 @@ func (s *Scheduler) trackChanges(oldRecord *core.Record, newData *domain.Domain,
})
}
// Check provider changes
providers := []struct {
field string
value string
}{
{"dns_provider", newData.DNSProvider},
{"hosting_provider", newData.HostingProvider},
{"email_provider", newData.EmailProvider},
{"ca_provider", newData.CAProvider},
}
for _, p := range providers {
oldVal := oldRecord.GetString(p.field)
if p.value != "" && p.value != oldVal && oldVal != "" {
history = append(history, domain.DomainHistory{
ChangeType: domain.ChangeTypeProvider,
FieldName: p.field,
OldValue: oldVal,
NewValue: p.value,
CreatedAt: now,
})
}
}
// Check security changes
if newData.PrivacyEnabled != oldRecord.GetBool("privacy_enabled") {
history = append(history, domain.DomainHistory{
ChangeType: domain.ChangeTypeSecurity,
FieldName: "privacy_enabled",
OldValue: fmt.Sprintf("%t", oldRecord.GetBool("privacy_enabled")),
NewValue: fmt.Sprintf("%t", newData.PrivacyEnabled),
CreatedAt: now,
})
}
if newData.TransferLock != oldRecord.GetBool("transfer_lock") {
history = append(history, domain.DomainHistory{
ChangeType: domain.ChangeTypeSecurity,
FieldName: "transfer_lock",
OldValue: fmt.Sprintf("%t", oldRecord.GetBool("transfer_lock")),
NewValue: fmt.Sprintf("%t", newData.TransferLock),
CreatedAt: now,
})
}
// Check host country code change
oldCountryCode := oldRecord.GetString("host_country_code")
if newData.HostCountryCode != "" && newData.HostCountryCode != oldCountryCode && oldCountryCode != "" {
history = append(history, domain.DomainHistory{
ChangeType: domain.ChangeTypeHost,
FieldName: "host_country_code",
OldValue: oldCountryCode,
NewValue: newData.HostCountryCode,
CreatedAt: now,
})
}
return history
}
+407 -35
View File
@@ -18,6 +18,7 @@ import (
"time"
"github.com/henrygd/beszel/internal/entities/domain"
"github.com/henrygd/beszel/internal/hub/domains/detect"
)
// LookupService handles WHOIS lookups with multiple fallback methods
@@ -34,14 +35,22 @@ func NewLookupService(apiKey string) *LookupService {
}
}
// LookupDomain performs a comprehensive domain lookup (WHOIS, DNS, SSL, Host)
// LookupDomain performs a comprehensive domain lookup (WHOIS, DNS, SSL, Host, Headers, SEO)
func (s *LookupService) LookupDomain(ctx context.Context, domainName string) (*domain.Domain, error) {
// Clean domain name
domainName = cleanDomain(domainName)
// Extract TLD
parts := strings.Split(domainName, ".")
tld := ""
if len(parts) >= 2 {
tld = strings.ToLower(parts[len(parts)-1])
}
// Initialize domain struct
d := &domain.Domain{
DomainName: domainName,
TLD: tld,
Active: true,
AlertDaysBefore: 30, // Default: alert 30 days before expiry
Tags: []string{},
@@ -50,25 +59,38 @@ func (s *LookupService) LookupDomain(ctx context.Context, domainName string) (*d
TXTRecords: []string{},
IPv4Addresses: []string{},
IPv6Addresses: []string{},
Headers: []domain.Header{},
Certificates: []domain.Certificate{},
DomainStatuses: []string{},
}
// Perform WHOIS lookup
whoisData, err := s.LookupWHOIS(ctx, domainName)
whoisData, rawWhois, err := s.LookupWHOIS(ctx, domainName)
if err == nil && whoisData != nil {
s.applyWHOISData(d, whoisData)
d.WHOISRaw = rawWhois
}
// Perform DNS lookups
s.lookupDNS(ctx, domainName, d)
// Perform SSL lookup
s.lookupSSL(ctx, domainName, d)
// Perform SSL lookup (certificate chain)
s.lookupCertificateChain(ctx, domainName, d)
// Perform host lookup (using first IPv4)
if len(d.IPv4Addresses) > 0 {
s.lookupHost(d.IPv4Addresses[0], d)
}
// Fetch HTTP headers for provider detection
s.lookupHeaders(ctx, domainName, d)
// Fetch SEO metadata
s.lookupSEO(ctx, domainName, d)
// Detect providers from gathered data
s.detectProviders(d)
// Fetch favicon
d.FaviconURL = fmt.Sprintf("https://www.google.com/s2/favicons?domain=%s&sz=128", domainName)
@@ -77,31 +99,31 @@ func (s *LookupService) LookupDomain(ctx context.Context, domainName string) (*d
}
// LookupWHOIS performs WHOIS lookup with multiple fallback methods
func (s *LookupService) LookupWHOIS(ctx context.Context, domainName string) (*domain.WHOISData, error) {
func (s *LookupService) LookupWHOIS(ctx context.Context, domainName string) (*domain.WHOISData, string, error) {
var lastErr error
// Try RDAP first
data, err := s.tryRDAP(ctx, domainName)
if err == nil && data != nil && hasValidData(data) {
return data, nil
return data, "", nil
}
if err != nil {
lastErr = err
}
// Try TCP WHOIS (this should work for .eu domains)
data, err = s.tryTCPWHOIS(ctx, domainName)
data, raw, err := s.tryTCPWHOIS(ctx, domainName)
if err == nil && data != nil && hasValidData(data) {
return data, nil
return data, raw, nil
}
if err != nil {
lastErr = err
}
// Try native whois command (often works when TCP fails)
data, err = s.tryNativeWHOIS(ctx, domainName)
data, raw, err = s.tryNativeWHOIS(ctx, domainName)
if err == nil && data != nil && hasValidData(data) {
return data, nil
return data, raw, nil
}
if err != nil {
lastErr = err
@@ -112,7 +134,7 @@ func (s *LookupService) LookupWHOIS(ctx context.Context, domainName string) (*do
if len(parts) >= 2 && strings.ToLower(parts[len(parts)-1]) == "eu" {
data, err = s.tryEURidWebScraping(ctx, domainName)
if err == nil && data != nil && hasValidData(data) {
return data, nil
return data, "", nil
}
if err != nil {
lastErr = err
@@ -121,7 +143,7 @@ func (s *LookupService) LookupWHOIS(ctx context.Context, domainName string) (*do
// Try alternative WHOIS services for .eu domains
data, err = s.tryAlternativeWHOIS(ctx, domainName)
if err == nil && data != nil && hasValidData(data) {
return data, nil
return data, "", nil
}
if err != nil {
lastErr = err
@@ -132,11 +154,11 @@ func (s *LookupService) LookupWHOIS(ctx context.Context, domainName string) (*do
if s.whoisXMLAPIKey != "" {
data, err = s.tryWhoisXML(ctx, domainName)
if err == nil && data != nil {
return data, nil
return data, "", nil
}
}
return nil, fmt.Errorf("all WHOIS lookup methods failed for %s: %w", domainName, lastErr)
return nil, "", fmt.Errorf("all WHOIS lookup methods failed for %s: %w", domainName, lastErr)
}
// tryRDAP attempts RDAP lookup
@@ -268,11 +290,11 @@ func (s *LookupService) tryRDAP(ctx context.Context, domainName string) (*domain
}
// tryNativeWHOIS tries the native whois command
func (s *LookupService) tryNativeWHOIS(ctx context.Context, domainName string) (*domain.WHOISData, error) {
func (s *LookupService) tryNativeWHOIS(ctx context.Context, domainName string) (*domain.WHOISData, string, error) {
// Check if whois command exists
_, err := exec.LookPath("whois")
if err != nil {
return nil, fmt.Errorf("whois command not found")
return nil, "", fmt.Errorf("whois command not found")
}
// Use longer timeout for .eu domains
@@ -289,10 +311,12 @@ func (s *LookupService) tryNativeWHOIS(ctx context.Context, domainName string) (
cmd := exec.CommandContext(cmdCtx, "whois", domainName)
output, err := cmd.Output()
if err != nil {
return nil, err
return nil, "", err
}
return s.parseWHOISOutput(string(output), domainName)
outStr := string(output)
data, err := s.parseWHOISOutput(outStr, domainName)
return data, outStr, err
}
// whoisServers maps common TLDs to their WHOIS servers
@@ -328,10 +352,10 @@ var whoisServers = map[string]string{
}
// tryTCPWHOIS performs WHOIS lookup via direct TCP connection (port 43)
func (s *LookupService) tryTCPWHOIS(ctx context.Context, domainName string) (*domain.WHOISData, error) {
func (s *LookupService) tryTCPWHOIS(ctx context.Context, domainName string) (*domain.WHOISData, string, error) {
parts := strings.Split(domainName, ".")
if len(parts) < 2 {
return nil, fmt.Errorf("invalid domain format")
return nil, "", fmt.Errorf("invalid domain format")
}
tld := strings.ToLower(parts[len(parts)-1])
@@ -352,19 +376,19 @@ func (s *LookupService) tryTCPWHOIS(ctx context.Context, domainName string) (*do
dialer := &net.Dialer{Timeout: timeout}
conn, err := dialer.DialContext(ctx, "tcp", addr)
if err != nil {
return nil, fmt.Errorf("tcp whois dial failed: %w", err)
return nil, "", fmt.Errorf("tcp whois dial failed: %w", err)
}
defer conn.Close()
// Some servers require the domain followed by \r\n
query := domainName + "\r\n"
if _, err := conn.Write([]byte(query)); err != nil {
return nil, fmt.Errorf("tcp whois write failed: %w", err)
return nil, "", fmt.Errorf("tcp whois write failed: %w", err)
}
// Read response with deadline
if err := conn.SetReadDeadline(time.Now().Add(timeout)); err != nil {
return nil, err
return nil, "", err
}
var output strings.Builder
@@ -379,7 +403,8 @@ func (s *LookupService) tryTCPWHOIS(ctx context.Context, domainName string) (*do
}
}
return s.parseWHOISOutput(output.String(), domainName)
data, err := s.parseWHOISOutput(output.String(), domainName)
return data, output.String(), err
}
// tryWhoisXML tries the WhoisXML API
@@ -1251,7 +1276,7 @@ func splitHex(value string) []string {
// lookupHost fetches host/geolocation info
func (s *LookupService) lookupHost(ip string, d *domain.Domain) {
// Use ip-api.com (free, no auth required for non-commercial use)
url := fmt.Sprintf("http://ip-api.com/json/%s?fields=status,message,country,regionName,city,lat,lon,isp,org,as", ip)
url := fmt.Sprintf("http://ip-api.com/json/%s?fields=status,message,country,countryCode,regionName,city,lat,lon,isp,org,as", ip)
client := &http.Client{Timeout: 5 * time.Second}
resp, err := client.Get(url)
@@ -1261,16 +1286,17 @@ func (s *LookupService) lookupHost(ip string, d *domain.Domain) {
defer resp.Body.Close()
var result struct {
Status string `json:"status"`
Message string `json:"message"`
Country string `json:"country"`
Region string `json:"regionName"`
City string `json:"city"`
Lat float64 `json:"lat"`
Lon float64 `json:"lon"`
ISP string `json:"isp"`
Org string `json:"org"`
AS string `json:"as"`
Status string `json:"status"`
Message string `json:"message"`
Country string `json:"country"`
CountryCode string `json:"countryCode"`
Region string `json:"regionName"`
City string `json:"city"`
Lat float64 `json:"lat"`
Lon float64 `json:"lon"`
ISP string `json:"isp"`
Org string `json:"org"`
AS string `json:"as"`
}
if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
@@ -1279,6 +1305,7 @@ func (s *LookupService) lookupHost(ip string, d *domain.Domain) {
if result.Status == "success" {
d.HostCountry = result.Country
d.HostCountryCode = result.CountryCode
d.HostRegion = result.Region
d.HostCity = result.City
d.HostLat = result.Lat
@@ -1301,6 +1328,27 @@ func (s *LookupService) applyWHOISData(d *domain.Domain, whois *domain.WHOISData
d.RegistrarID = whois.Registrar.ID
d.RegistrarURL = whois.Registrar.URL
d.RegistryDomainID = whois.Registrar.RegistryDomainID
d.DomainStatuses = whois.Status
// Detect privacy protection from registrant name
registrantLower := strings.ToLower(whois.Registrant.Name + " " + whois.Registrant.Organization)
d.PrivacyEnabled = strings.Contains(registrantLower, "redacted") ||
strings.Contains(registrantLower, "privacy") ||
strings.Contains(registrantLower, "whoisguard") ||
strings.Contains(registrantLower, "not disclosed") ||
strings.Contains(registrantLower, "hidden") ||
strings.Contains(registrantLower, "data protected") ||
strings.Contains(registrantLower, "gdpr") ||
strings.Contains(registrantLower, "data redacted")
// Detect transfer lock from statuses
for _, status := range whois.Status {
statusLower := strings.ToLower(status)
if strings.Contains(statusLower, "clienttransferprohibited") || strings.Contains(statusLower, "servertransferprohibited") {
d.TransferLock = true
break
}
}
// Apply registrant contact info if available
if whois.Registrant.Name != "" || whois.Registrant.Organization != "" {
@@ -1337,6 +1385,330 @@ func cleanDomain(domain string) string {
return strings.ToLower(strings.TrimSpace(domain))
}
// lookupCertificateChain fetches the full TLS certificate chain
func (s *LookupService) lookupCertificateChain(ctx context.Context, domainName string, d *domain.Domain) {
conn, err := tls.DialWithDialer(&net.Dialer{Timeout: 5 * time.Second}, "tcp", domainName+":443", &tls.Config{
ServerName: domainName,
InsecureSkipVerify: true,
})
if err != nil {
return
}
defer conn.Close()
certs := conn.ConnectionState().PeerCertificates
for i, cert := range certs {
issuer := ""
if len(cert.Issuer.Organization) > 0 {
issuer = cert.Issuer.Organization[0]
} else if cert.Issuer.CommonName != "" {
issuer = cert.Issuer.CommonName
}
altNames := make([]string, 0, len(cert.DNSNames)+len(cert.IPAddresses)+len(cert.EmailAddresses))
altNames = append(altNames, cert.DNSNames...)
for _, ip := range cert.IPAddresses {
altNames = append(altNames, ip.String())
}
for _, email := range cert.EmailAddresses {
altNames = append(altNames, email)
}
// For leaf cert, also set legacy SSL fields
if i == 0 {
if len(cert.Issuer.Organization) > 0 {
d.SSLIssuer = cert.Issuer.Organization[0]
}
if len(cert.Issuer.Country) > 0 {
d.SSLIssuerCountry = cert.Issuer.Country[0]
}
d.SSLValidFrom = &cert.NotBefore
d.SSLValidTo = &cert.NotAfter
d.SSLSubject = cert.Subject.CommonName
fingerprint := sha256.Sum256(cert.Raw)
d.SSLFingerprint = strings.ToUpper(strings.Join(splitHex(hex.EncodeToString(fingerprint[:])), ":"))
d.SSLSignatureAlgo = cert.SignatureAlgorithm.String()
switch key := cert.PublicKey.(type) {
case *rsa.PublicKey:
d.SSLKeySize = key.N.BitLen()
case *ecdsa.PublicKey:
d.SSLKeySize = key.Curve.Params().BitSize
default:
d.SSLKeySize = 0
}
}
caProvider := detect.DetectCertificateAuthority(issuer)
d.Certificates = append(d.Certificates, domain.Certificate{
Issuer: issuer,
Subject: cert.Subject.CommonName,
AltNames: altNames,
ValidFrom: cert.NotBefore,
ValidTo: cert.NotAfter,
CAProvider: caProvider,
})
}
// Set top-level CA provider from the chain
if len(d.Certificates) > 0 {
d.CAProvider = d.Certificates[len(d.Certificates)-1].CAProvider
}
}
// lookupHeaders fetches HTTP response headers
func (s *LookupService) lookupHeaders(ctx context.Context, domainName string, d *domain.Domain) {
client := &http.Client{
Timeout: 10 * time.Second,
CheckRedirect: func(req *http.Request, via []*http.Request) error {
if len(via) >= 5 {
return fmt.Errorf("too many redirects")
}
return nil
},
}
url := "https://" + domainName
req, err := http.NewRequestWithContext(ctx, "HEAD", url, nil)
if err != nil {
return
}
req.Header.Set("User-Agent", "Mozilla/5.0 (compatible; Beszel/1.0; +https://github.com/henrygd/beszel)")
resp, err := client.Do(req)
if err != nil {
// Try HTTP fallback
req, err = http.NewRequestWithContext(ctx, "HEAD", "http://"+domainName, nil)
if err != nil {
return
}
req.Header.Set("User-Agent", "Mozilla/5.0 (compatible; Beszel/1.0; +https://github.com/henrygd/beszel)")
resp, err = client.Do(req)
if err != nil {
return
}
}
defer resp.Body.Close()
for name, values := range resp.Header {
for _, value := range values {
d.Headers = append(d.Headers, domain.Header{
Name: strings.ToLower(name),
Value: value,
})
}
}
}
// lookupSEO fetches and parses SEO metadata
func (s *LookupService) lookupSEO(ctx context.Context, domainName string, d *domain.Domain) {
client := &http.Client{
Timeout: 10 * time.Second,
CheckRedirect: func(req *http.Request, via []*http.Request) error {
if len(via) >= 5 {
return fmt.Errorf("too many redirects")
}
return nil
},
}
// Fetch HTML
url := "https://" + domainName
req, err := http.NewRequestWithContext(ctx, "GET", url, nil)
if err != nil {
return
}
req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36")
req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8")
resp, err := client.Do(req)
if err != nil {
return
}
defer resp.Body.Close()
// Limit reading to avoid large responses
body, err := io.ReadAll(io.LimitReader(resp.Body, 512*1024))
if err != nil {
return
}
html := string(body)
seo := &domain.SEOMeta{
OpenGraph: domain.OpenGraphMeta{},
Twitter: domain.TwitterMeta{},
General: domain.GeneralMeta{},
Robots: domain.RobotsTxt{Fetched: false, Groups: []domain.RobotsGroup{}, Sitemaps: []string{}},
}
// Parse general meta tags
seo.General.Title = extractMetaTag(html, "title")
seo.General.Description = extractMetaTag(html, "description")
seo.General.Author = extractMetaTag(html, "author")
seo.General.Robots = extractMetaTag(html, "robots")
seo.General.Keywords = extractMetaTag(html, "keywords")
seo.General.Canonical = extractLinkRel(html, "canonical")
// Parse Open Graph
seo.OpenGraph.URL = extractMetaProperty(html, "og:url")
seo.OpenGraph.Type = extractMetaProperty(html, "og:type")
seo.OpenGraph.Title = extractMetaProperty(html, "og:title")
seo.OpenGraph.Description = extractMetaProperty(html, "og:description")
seo.OpenGraph.Images = extractMetaProperties(html, "og:image")
// Parse Twitter
seo.Twitter.Title = extractMetaProperty(html, "twitter:title")
seo.Twitter.Description = extractMetaProperty(html, "twitter:description")
seo.Twitter.Image = extractMetaProperty(html, "twitter:image")
seo.Twitter.Card = extractMetaProperty(html, "twitter:card")
// Fetch robots.txt
robotsURL := url + "/robots.txt"
robotsReq, err := http.NewRequestWithContext(ctx, "GET", robotsURL, nil)
if err == nil {
robotsResp, err := client.Do(robotsReq)
if err == nil && robotsResp.StatusCode >= 200 && robotsResp.StatusCode < 300 {
robotsBody, err := io.ReadAll(io.LimitReader(robotsResp.Body, 256*1024))
robotsResp.Body.Close()
if err == nil {
seo.Robots = parseRobotsTxt(string(robotsBody))
}
} else if robotsResp != nil {
robotsResp.Body.Close()
}
}
d.SEOMeta = seo
}
// extractMetaTag extracts a meta tag by name attribute
func extractMetaTag(html, name string) string {
// Match <meta name="xxx" content="yyy"> or <meta name='xxx' content='yyy'>
re := regexp.MustCompile(`(?i)<meta\s+name=["']` + regexp.QuoteMeta(name) + `["']\s+content=["']([^"']*)["']`)
match := re.FindStringSubmatch(html)
if len(match) > 1 {
return match[1]
}
// Try reverse order
re = regexp.MustCompile(`(?i)<meta\s+content=["']([^"']*)["']\s+name=["']` + regexp.QuoteMeta(name) + `["']`)
match = re.FindStringSubmatch(html)
if len(match) > 1 {
return match[1]
}
return ""
}
// extractMetaProperty extracts a meta tag by property attribute
func extractMetaProperty(html, prop string) string {
re := regexp.MustCompile(`(?i)<meta\s+property=["']` + regexp.QuoteMeta(prop) + `["']\s+content=["']([^"']*)["']`)
match := re.FindStringSubmatch(html)
if len(match) > 1 {
return match[1]
}
// Try reverse order
re = regexp.MustCompile(`(?i)<meta\s+content=["']([^"']*)["']\s+property=["']` + regexp.QuoteMeta(prop) + `["']`)
match = re.FindStringSubmatch(html)
if len(match) > 1 {
return match[1]
}
return ""
}
// extractMetaProperties extracts all meta tags matching a property prefix
func extractMetaProperties(html, prop string) []string {
re := regexp.MustCompile(`(?i)<meta\s+property=["']` + regexp.QuoteMeta(prop) + `["']\s+content=["']([^"']*)["']`)
matches := re.FindAllStringSubmatch(html, -1)
var results []string
for _, match := range matches {
if len(match) > 1 {
results = append(results, match[1])
}
}
return results
}
// extractLinkRel extracts a link rel href value
func extractLinkRel(html, rel string) string {
re := regexp.MustCompile(`(?i)<link\s+rel=["']` + regexp.QuoteMeta(rel) + `["']\s+href=["']([^"']*)["']`)
match := re.FindStringSubmatch(html)
if len(match) > 1 {
return match[1]
}
return ""
}
// parseRobotsTxt parses robots.txt content
func parseRobotsTxt(content string) domain.RobotsTxt {
result := domain.RobotsTxt{
Fetched: true,
Groups: []domain.RobotsGroup{},
Sitemaps: []string{},
}
lines := strings.Split(content, "\n")
var currentGroup *domain.RobotsGroup
for _, line := range lines {
line = strings.TrimSpace(line)
if line == "" || strings.HasPrefix(line, "#") {
continue
}
parts := strings.SplitN(line, ":", 2)
if len(parts) < 2 {
continue
}
key := strings.TrimSpace(strings.ToLower(parts[0]))
value := strings.TrimSpace(parts[1])
switch key {
case "user-agent":
if currentGroup != nil {
result.Groups = append(result.Groups, *currentGroup)
}
currentGroup = &domain.RobotsGroup{
UserAgents: []string{value},
Rules: []domain.RobotsRule{},
}
case "allow", "disallow":
if currentGroup == nil {
currentGroup = &domain.RobotsGroup{
UserAgents: []string{"*"},
Rules: []domain.RobotsRule{},
}
}
currentGroup.Rules = append(currentGroup.Rules, domain.RobotsRule{
Type: key,
Value: value,
})
case "sitemap":
result.Sitemaps = append(result.Sitemaps, value)
}
}
if currentGroup != nil {
result.Groups = append(result.Groups, *currentGroup)
}
return result
}
// detectProviders detects DNS, hosting, email, and CA providers
func (s *LookupService) detectProviders(d *domain.Domain) {
d.DNSProvider = detect.DetectDNSProvider(d.NameServers)
d.EmailProvider = detect.DetectEmailProvider(d.MXRecords)
if len(d.Headers) > 0 {
headerMap := make(http.Header)
for _, h := range d.Headers {
headerMap.Add(h.Name, h.Value)
}
d.HostingProvider = detect.DetectHostingProvider(headerMap)
}
}
// hasValidData checks if WHOIS data has useful parsed fields
func hasValidData(data *domain.WHOISData) bool {
if data == nil {
+60
View File
@@ -7,6 +7,7 @@ import (
"time"
"github.com/henrygd/beszel/internal/entities/monitor"
"github.com/henrygd/beszel/internal/hub/pagespeed"
"github.com/pocketbase/dbx"
"github.com/pocketbase/pocketbase/core"
)
@@ -50,6 +51,7 @@ func (h *APIHandler) RegisterRoutes(se *core.ServeEvent) {
api.POST("/{id}/resume", h.resumeMonitor)
api.GET("/{id}/stats", h.getStats)
api.GET("/{id}/heartbeats", h.getHeartbeats)
api.POST("/{id}/pagespeed", h.runPageSpeedCheck)
}
// HeartbeatSummary represents a minimal heartbeat for the monitor list
@@ -609,6 +611,64 @@ func (h *APIHandler) getStats(e *core.RequestEvent) error {
})
}
// runPageSpeedCheck runs a PageSpeed Insights check for a monitor
func (h *APIHandler) runPageSpeedCheck(e *core.RequestEvent) error {
id := e.Request.PathValue("id")
if id == "" {
return e.BadRequestError("Monitor ID is required", nil)
}
record, err := h.app.FindRecordById("monitors", id)
if err != nil {
return e.NotFoundError("Monitor not found", err)
}
if record.GetString("user") != e.Auth.Id {
return e.ForbiddenError("Access denied", nil)
}
url := record.GetString("url")
if url == "" {
return e.BadRequestError("Monitor does not have a URL", nil)
}
// Get strategy from query param, default to mobile
strategy := e.Request.URL.Query().Get("strategy")
if strategy == "" {
strategy = "mobile"
}
if strategy != "mobile" && strategy != "desktop" {
return e.BadRequestError("strategy must be 'mobile' or 'desktop'", nil)
}
checker := pagespeed.NewChecker("")
metrics, err := checker.CheckURL(url, strategy)
if err != nil {
return e.InternalServerError("PageSpeed check failed", err)
}
vitals := pagespeed.GetCoreWebVitalsStatus(metrics)
return e.JSON(http.StatusOK, map[string]interface{}{
"performance": metrics.Performance,
"accessibility": metrics.Accessibility,
"bestPractices": metrics.BestPractices,
"seo": metrics.SEO,
"pwa": metrics.PWA,
"fcp": metrics.FCP,
"lcp": metrics.LCP,
"ttfb": metrics.TTFB,
"cls": metrics.CLS,
"tbt": metrics.TBT,
"speedIndex": metrics.SpeedIndex,
"tti": metrics.TTI,
"strategy": metrics.Strategy,
"checkedAt": metrics.CheckedAt,
"url": metrics.URL,
"vitals": vitals,
})
}
// getHeartbeats returns recent heartbeats for a monitor
func (h *APIHandler) getHeartbeats(e *core.RequestEvent) error {
id := e.Request.PathValue("id")
+29 -18
View File
@@ -5,6 +5,7 @@ import (
"fmt"
"io"
"net/http"
"net/url"
"time"
)
@@ -65,21 +66,21 @@ type PageSpeedResponse struct {
// Metrics represents the extracted performance metrics
type Metrics struct {
Performance float64 `json:"performance"`
Accessibility float64 `json:"accessibility"`
BestPractices float64 `json:"bestPractices"`
SEO float64 `json:"seo"`
PWA float64 `json:"pwa"`
FCP float64 `json:"fcp"` // First Contentful Paint (ms)
LCP float64 `json:"lcp"` // Largest Contentful Paint (ms)
TTFB float64 `json:"ttfb"` // Time to First Byte (ms)
CLS float64 `json:"cls"` // Cumulative Layout Shift
TBT float64 `json:"tbt"` // Total Blocking Time (ms)
SpeedIndex float64 `json:"speedIndex"` // Speed Index (ms)
TTI float64 `json:"tti"` // Time to Interactive (ms)
CheckedAt time.Time `json:"checkedAt"`
URL string `json:"url"`
Strategy string `json:"strategy"` // mobile or desktop
Performance float64 `json:"performance"`
Accessibility float64 `json:"accessibility"`
BestPractices float64 `json:"bestPractices"`
SEO float64 `json:"seo"`
PWA float64 `json:"pwa"`
FCP float64 `json:"fcp"` // First Contentful Paint (ms)
LCP float64 `json:"lcp"` // Largest Contentful Paint (ms)
TTFB float64 `json:"ttfb"` // Time to First Byte (ms)
CLS float64 `json:"cls"` // Cumulative Layout Shift
TBT float64 `json:"tbt"` // Total Blocking Time (ms)
SpeedIndex float64 `json:"speedIndex"` // Speed Index (ms)
TTI float64 `json:"tti"` // Time to Interactive (ms)
CheckedAt time.Time `json:"checkedAt"`
URL string `json:"url"`
Strategy string `json:"strategy"` // mobile or desktop
}
// Checker handles PageSpeed checks
@@ -99,7 +100,7 @@ func NewChecker(apiKey string) *Checker {
}
// CheckURL runs a PageSpeed check on a URL
func (c *Checker) CheckURL(url string, strategy string) (*Metrics, error) {
func (c *Checker) CheckURL(pageURL string, strategy string) (*Metrics, error) {
if strategy == "" {
strategy = "mobile"
}
@@ -107,7 +108,7 @@ func (c *Checker) CheckURL(url string, strategy string) (*Metrics, error) {
// Build PageSpeed API URL
apiURL := fmt.Sprintf(
"https://www.googleapis.com/pagespeedonline/v5/runPagespeed?url=%s&strategy=%s&category=PERFORMANCE&category=ACCESSIBILITY&category=BEST_PRACTICES&category=SEO&category=PWA",
url,
url.QueryEscape(pageURL),
strategy,
)
@@ -132,7 +133,7 @@ func (c *Checker) CheckURL(url string, strategy string) (*Metrics, error) {
}
metrics := &Metrics{
URL: url,
URL: pageURL,
Strategy: strategy,
CheckedAt: time.Now(),
Performance: result.LighthouseResult.Categories.Performance.Score * 100,
@@ -192,6 +193,7 @@ func GetCoreWebVitalsStatus(metrics *Metrics) map[string]string {
"cls": getCLSStatus(metrics.CLS),
"fcp": getFCPStatus(metrics.FCP),
"ttfb": getTTFBStatus(metrics.TTFB),
"tti": getTTIStatus(metrics.TTI),
}
}
@@ -240,6 +242,15 @@ func getTTFBStatus(value float64) string {
return "poor"
}
func getTTIStatus(value float64) string {
if value <= 3800 {
return "good"
} else if value <= 7300 {
return "needs-improvement"
}
return "poor"
}
// FormatDuration formats milliseconds to readable string
func FormatDuration(ms float64) string {
if ms < 1000 {