package services import ( "encoding/json" "fmt" "io" "log" "math/rand" "net/http" "net/url" "os" "path/filepath" "strconv" "strings" "sync/atomic" "time" "fotbal-club/internal/config" ) // StartPrefetcher starts a background job that periodically fetches // RefreshYouTubeChannelNow triggers an immediate refresh of the YouTube channel cache. // It downloads the channel videos JSON from the external API and stores it on disk. // Returns an error only in case of a fetch/write failure. func RefreshYouTubeChannelNow(channelURL string) error { ch := strings.TrimSpace(channelURL) if ch == "" { return fmt.Errorf("empty youtube channel url") } return fetchYouTubeChannel(ch) } // RefreshZoneramaNow triggers an immediate refresh of Zonerama cache given a direct link. // The link should be a Zonerama profile or album URL. func RefreshZoneramaNow(link string) error { l := strings.TrimSpace(link) if l == "" { return fmt.Errorf("empty zonerama link") } return fetchZonerama(l) } // fetchZoneramaOnce reads zonerama_url or a zonerama-looking gallery_url from cached settings // and updates the zonerama cache (metadata JSON + images) if available. func fetchZoneramaOnce() error { cacheDir := filepath.Join("cache", "prefetch") b, err := os.ReadFile(filepath.Join(cacheDir, "settings.json")) if err != nil { return fmt.Errorf("settings cache missing: %w", err) } var s struct { Zonerama string `json:"zonerama_url"` Gallery string `json:"gallery_url"` } if err := json.Unmarshal(b, &s); err != nil { return fmt.Errorf("invalid settings cache: %w", err) } link := strings.TrimSpace(s.Zonerama) if link == "" { g := strings.TrimSpace(s.Gallery) if strings.Contains(strings.ToLower(g), "zonerama.com") { link = g } } if link == "" { return fmt.Errorf("zonerama link not configured") } return fetchZonerama(link) } // fetchZonerama calls the external scraper API and fetches the profile (album list metadata only) // then fetches each album with photos and stores them in zonerama_albums.json func fetchZonerama(link string) error { cacheDir := filepath.Join("cache", "prefetch") if err := os.MkdirAll(cacheDir, 0o755); err != nil { log.Printf("[prefetch] Zonerama ERROR: Failed to create cache dir: %v", err) return err } // Profile fetch - gets album metadata only (no photos) albumLimit := envInt("ZONERAMA_ALBUM_LIMIT", 10) // Fetch up to 10 albums metadata apiBase := "https://zonerama.tdvorak.dev/zonerama?link=" + url.QueryEscape(strings.TrimSpace(link)) + "&album_limit=" + strconv.Itoa(albumLimit) + "&photo_limit=0" log.Printf("[prefetch] Fetching Zonerama profile: %s (album_limit=%d, no photos)", apiBase, albumLimit) // Increase timeout to 60s since the API can take longer to fetch client := &http.Client{Timeout: 60 * time.Second} // Retry logic with exponential backoff (3 attempts) var resp *http.Response var err error maxRetries := 3 for attempt := 1; attempt <= maxRetries; attempt++ { var req *http.Request req, err = http.NewRequest("GET", apiBase, nil) if err != nil { log.Printf("[prefetch] Zonerama ERROR: Failed to create request: %v", err) return err } req.Header.Set("User-Agent", "fotbal-club-prefetch/1.0") log.Printf("[prefetch] Zonerama: Attempt %d/%d...", attempt, maxRetries) resp, err = client.Do(req) if err == nil && resp.StatusCode == 200 { log.Printf("[prefetch] Zonerama: Success on attempt %d", attempt) break } if resp != nil { log.Printf("[prefetch] Zonerama: Attempt %d got status %d", attempt, resp.StatusCode) _ = resp.Body.Close() } if err != nil { log.Printf("[prefetch] Zonerama: Attempt %d error: %v", attempt, err) } if attempt < maxRetries { // Exponential backoff: 2s, 4s backoff := time.Duration(1<= 300 { return fmt.Errorf("zonerama profile api status %d after %d attempts", resp.StatusCode, maxRetries) } // Read response body bodyData, err := io.ReadAll(resp.Body) if err != nil { return fmt.Errorf("failed to read response: %w", err) } // Parse and add fetched_at timestamp var profileData map[string]interface{} if err := json.Unmarshal(bodyData, &profileData); err != nil { return fmt.Errorf("failed to parse profile JSON: %w", err) } profileData["fetched_at"] = time.Now().Format(time.RFC3339) // Write profile metadata to disk (album list, no photos) profilePath := filepath.Join(cacheDir, "zonerama_profile.json") if err := writeJSONAtomic(profilePath, profileData); err != nil { return fmt.Errorf("failed to write profile: %w", err) } log.Printf("[prefetch] Updated Zonerama profile cache") // Now fetch each album with photos and store them in zonerama_albums.json var profile struct { Albums []struct { ID string `json:"id"` URL string `json:"url"` } `json:"albums"` } if err := json.Unmarshal(bodyData, &profile); err != nil { log.Printf("[prefetch] Zonerama WARNING: Failed to parse albums from profile: %v", err) return nil // Don't fail completely, we saved the profile } if len(profile.Albums) == 0 { log.Printf("[prefetch] Zonerama: No albums found in profile") return nil } log.Printf("[prefetch] Zonerama: Fetching %d albums with photos...", len(profile.Albums)) // Fetch individual albums with photos photoLimit := envInt("ZONERAMA_PHOTO_LIMIT", 50) // Default 50 photos per album fetchedAlbums, err := fetchZoneramaAlbums(profile.Albums, photoLimit, client) if err != nil { log.Printf("[prefetch] Zonerama WARNING: Failed to fetch albums: %v", err) return nil // Don't fail completely } // Save albums to zonerama_albums.json albumsPath := filepath.Join(cacheDir, "zonerama_albums.json") if err := writeJSONAtomic(albumsPath, fetchedAlbums); err != nil { return fmt.Errorf("failed to write albums: %w", err) } log.Printf("[prefetch] Zonerama: Saved %d albums with photos", len(fetchedAlbums)) // Regenerate flat files for frontend consumption if err := RegenerateFlatGalleryFiles(); err != nil { log.Printf("[prefetch] Zonerama WARNING: Failed to regenerate flat files: %v", err) } return nil } // fetchZoneramaAlbums fetches individual albums with photos func fetchZoneramaAlbums(albums []struct { ID string `json:"id"` URL string `json:"url"` }, photoLimit int, client *http.Client) ([]interface{}, error) { type zoneramaPhoto struct { ID string `json:"id"` PageURL string `json:"page_url"` Image1500 string `json:"image_1500"` } type zoneramaAlbum struct { ID string `json:"id"` Title string `json:"title"` URL string `json:"url"` Date string `json:"date"` PhotosCount int `json:"photos_count"` ViewsCount int `json:"views_count"` Photos []zoneramaPhoto `json:"photos"` FetchedAt string `json:"fetched_at"` } var result []interface{} fetchedAt := time.Now().Format(time.RFC3339) for i, album := range albums { if album.URL == "" { log.Printf("[prefetch] Zonerama: Skipping album %d (no URL)", i) continue } // Fetch album with photos apiURL := fmt.Sprintf("https://zonerama.tdvorak.dev/zonerama-album?link=%s&photo_limit=%d", url.QueryEscape(album.URL), photoLimit) log.Printf("[prefetch] Zonerama: Fetching album %d/%d: %s", i+1, len(albums), album.URL) req, err := http.NewRequest("GET", apiURL, nil) if err != nil { log.Printf("[prefetch] Zonerama: Failed to create request for album %s: %v", album.ID, err) continue } req.Header.Set("User-Agent", "fotbal-club-prefetch/1.0") resp, err := client.Do(req) if err != nil { log.Printf("[prefetch] Zonerama: Failed to fetch album %s: %v", album.ID, err) continue } if resp.StatusCode != 200 { log.Printf("[prefetch] Zonerama: Album %s returned status %d", album.ID, resp.StatusCode) _ = resp.Body.Close() continue } var albumData zoneramaAlbum if err := json.NewDecoder(resp.Body).Decode(&albumData); err != nil { log.Printf("[prefetch] Zonerama: Failed to parse album %s: %v", album.ID, err) _ = resp.Body.Close() continue } _ = resp.Body.Close() albumData.FetchedAt = fetchedAt result = append(result, albumData) log.Printf("[prefetch] Zonerama: Album %s fetched with %d photos", albumData.ID, len(albumData.Photos)) // Small delay between requests to avoid overwhelming the API if i < len(albums)-1 { time.Sleep(500 * time.Millisecond) } } return result, nil } // downloadToFile downloads a URL to a local path via temp file and atomic rename. func downloadToFile(client *http.Client, src string, outPath string) error { req, err := http.NewRequest("GET", src, nil) if err != nil { return err } req.Header.Set("User-Agent", "fotbal-club-prefetch/1.0") resp, err := client.Do(req) if err != nil { return err } defer resp.Body.Close() if resp.StatusCode < 200 || resp.StatusCode >= 300 { return fmt.Errorf("download status %d", resp.StatusCode) } if err := os.MkdirAll(filepath.Dir(outPath), 0o755); err != nil { return err } tmp := outPath + ".tmp" f, err := os.Create(tmp) if err != nil { return err } if _, err := io.Copy(f, resp.Body); err != nil { _ = f.Close() _ = os.Remove(tmp) return err } if err := f.Close(); err != nil { _ = os.Remove(tmp) return err } return os.Rename(tmp, outPath) } // writeJSONAtomic writes v as pretty JSON atomically. func writeJSONAtomic(path string, v any) error { b, err := json.MarshalIndent(v, "", " ") if err != nil { return err } if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { return err } tmp := path + ".tmp" if err := os.WriteFile(tmp, b, 0o644); err != nil { return err } return os.Rename(tmp, path) } // --- small helpers for env parsing --- func envBool(name string, def bool) bool { v := strings.TrimSpace(os.Getenv(name)) if v == "" { return def } b, err := strconv.ParseBool(v) if err != nil { return def } return b } func envInt(name string, def int) int { v := strings.TrimSpace(os.Getenv(name)) if v == "" { return def } if n, err := strconv.Atoi(v); err == nil { return n } return def } // baseURL should be something like "http://127.0.0.1:8080/api/v1" func StartPrefetcher(baseURL string) { // Normalize baseURL (no trailing slash) baseURL = strings.TrimSuffix(baseURL, "/") cacheDir := filepath.Join("cache", "prefetch") if err := os.MkdirAll(cacheDir, 0o755); err != nil { log.Printf("[prefetch] failed to create cache dir: %v", err) } client := &http.Client{Timeout: 20 * time.Second} // Feature toggles (env) enableFastMatch := envBool("ENABLE_FAST_MATCH_PREFETCH", true) enableYouTube := envBool("ENABLE_YOUTUBE_PREFETCH", true) enableZonerama := envBool("ENABLE_ZONERAMA_PREFETCH", true) // Wait until the API is reachable to avoid a startup race where the first fetch fails waitForServer := func(healthURL string, timeout time.Duration) { deadline := time.Now().Add(timeout) for { if time.Now().After(deadline) { log.Printf("[prefetch] server not reachable within %s, continuing anyway", timeout) return } resp, err := client.Get(healthURL) if err == nil { _ = resp.Body.Close() if resp.StatusCode >= 200 && resp.StatusCode < 300 { return } } time.Sleep(500 * time.Millisecond) } } fetchAll := func() { doPrefetchCycleGuarded(client, baseURL) } // Run initial fetch shortly after server is up (best-effort) go func() { healthURL := baseURL + "/health" waitForServer(healthURL, 15*time.Second) fetchAll() }() // Then run every configurable interval (default 30 minutes) go func() { defer func() { if r := recover(); r != nil { log.Printf("[prefetch] panic in ticker goroutine: %v", r) } }() interval := 30 * time.Minute if v := strings.TrimSpace(os.Getenv("PREFETCH_INTERVAL_MINUTES")); v != "" { if mins, err := strconv.Atoi(v); err == nil && mins > 0 { interval = time.Duration(mins) * time.Minute } } ticker := time.NewTicker(interval) defer ticker.Stop() for range ticker.C { // add small jitter 0-20s to avoid synchronization in multi-instance setups j := time.Duration(rand.Intn(20)) * time.Second time.Sleep(j) fetchAll() } }() // During match windows, prefetch more frequently (every 5 minutes) if enableFastMatch { go func() { defer func() { if r := recover(); r != nil { log.Printf("[prefetch] panic in fast ticker goroutine: %v", r) } }() fast := time.NewTicker(5 * time.Minute) defer fast.Stop() for range fast.C { // Only fetch if we appear to be within an active match window if isDuringMatch(cacheDir) { // add small jitter 0-10s j := time.Duration(rand.Intn(10)) * time.Second time.Sleep(j) fetchAll() } } }() } else { log.Printf("[prefetch] fast match prefetch disabled via ENABLE_FAST_MATCH_PREFETCH=false") } // Separate daily randomized YouTube fetcher if enableYouTube { go func() { defer func() { if r := recover(); r != nil { log.Printf("[prefetch] panic in YouTube goroutine: %v", r) } }() // Perform one best-effort initial fetch a short time after startup time.Sleep(10 * time.Second) if err := fetchYouTubeChannelOnce(); err != nil { log.Printf("[prefetch] initial YouTube fetch skipped/failed: %v", err) } for { next := nextRandomDailyTime() d := time.Until(next) if d <= 0 { d = 10 * time.Minute } log.Printf("[prefetch] YouTube next run at %s (in %s)", next.Format(time.RFC3339), d.Truncate(time.Second)) t := time.NewTimer(d) <-t.C _ = fetchYouTubeChannelOnce() } }() } else { log.Printf("[prefetch] YouTube prefetch disabled via ENABLE_YOUTUBE_PREFETCH=false") } // Zonerama/Gallery fetcher: once daily (configurable) to avoid stressing the API if enableZonerama { go func() { defer func() { if r := recover(); r != nil { log.Printf("[prefetch] panic in Zonerama goroutine: %v", r) } }() // initial delay to let other caches warm up time.Sleep(15 * time.Second) // Perform initial fetch if err := fetchZoneramaOnce(); err != nil { log.Printf("[prefetch] initial Zonerama/Gallery fetch failed: %v", err) } // Default to daily (1440 minutes = 24 hours) to avoid stressing the API intervalMin := envInt("ZONERAMA_FETCH_INTERVAL_MINUTES", 1440) if intervalMin < 5 { intervalMin = 5 } log.Printf("[prefetch] Zonerama/Gallery will refresh every %d minutes (%s)", intervalMin, time.Duration(intervalMin)*time.Minute) for { // wait configured interval time.Sleep(time.Duration(intervalMin) * time.Minute) if err := fetchZoneramaOnce(); err != nil { // log only informative message; do not spam log.Printf("[prefetch] Zonerama/Gallery fetch skipped/failed: %v", err) } } }() } else { log.Printf("[prefetch] Zonerama/Gallery prefetch disabled via ENABLE_ZONERAMA_PREFETCH=false") } } // PrefetchOnce runs a single prefetch cycle immediately. Useful to trigger after setup. func PrefetchOnce(baseURL string) { baseURL = strings.TrimSuffix(baseURL, "/") client := &http.Client{Timeout: 20 * time.Second} doPrefetchCycleGuarded(client, baseURL) } // doPrefetchCycle performs one full prefetch cycle: static public endpoints + dynamic FACR endpoints. func doPrefetchCycle(client *http.Client, baseURL string) { cacheDir := filepath.Join("cache", "prefetch") _ = os.MkdirAll(cacheDir, 0o755) start := time.Now() // Track endpoint fetch statuses for admin/debugging payload type epStatus struct { Path string `json:"path"` File string `json:"file"` Ok bool `json:"ok"` Error string `json:"error,omitempty"` } var statuses []epStatus // 1) Static public endpoints endpoints := map[string]string{ "/settings": "settings.json", "/seo": "seo.json", "/articles?page=1&page_size=10&published=true": "articles.json", "/sponsors": "sponsors.json", "/events/upcoming": "events_upcoming.json", "/public/team-logo-overrides": "team_logo_overrides.json", "/competition-aliases": "competition_aliases.json", } statuses = make([]epStatus, 0, len(endpoints)) for path, file := range endpoints { url := baseURL + path outPath := filepath.Join(cacheDir, file) log.Printf("[prefetch] Fetching %s -> %s", url, outPath) if err := fetchToFile(client, url, outPath); err != nil { log.Printf("[prefetch] ERROR: %s -> %s failed: %v", path, file, err) statuses = append(statuses, epStatus{Path: path, File: file, Ok: false, Error: err.Error()}) } else { log.Printf("[prefetch] SUCCESS: updated %s", file) statuses = append(statuses, epStatus{Path: path, File: file, Ok: true}) } } // Prepare matches.json from FACR data if available; fallback to events_upcoming.json createdMatches := false buildMatchesFromFACR := func(data []byte) bool { type facrMatch struct { DateTime string `json:"date_time"` Home string `json:"home"` Away string `json:"away"` Venue string `json:"venue"` HomeLogoURL string `json:"home_logo_url"` AwayLogoURL string `json:"away_logo_url"` MatchID string `json:"match_id"` } var facr struct { Competitions []struct { Name string `json:"name"` Code string `json:"code"` Matches []facrMatch `json:"matches"` } `json:"competitions"` } if err := json.Unmarshal(data, &facr); err != nil { return false } // Collect upcoming matches (next 30 days) in simplified format now := time.Now() max := now.Add(30 * 24 * time.Hour) var out []map[string]any for _, c := range facr.Competitions { compName := strings.TrimSpace(c.Name) if compName == "" { compName = strings.TrimSpace(c.Code) } for _, m := range c.Matches { dt := strings.TrimSpace(m.DateTime) if dt == "" { continue } // dt like "12.08.2023 18:00" parts := strings.SplitN(dt, " ", 2) d := parts[0] t := "" if len(parts) > 1 { t = parts[1] } // parse date dd.mm.yyyy dd := strings.Split(d, ".") if len(dd) < 3 { continue } day := dd[0] month := dd[1] year := dd[2] if len(month) == 1 { month = "0" + month } if len(day) == 1 { day = "0" + day } isoDate := year + "-" + month + "-" + day if len(t) >= 5 { t = t[:5] } else { t = "18:00" } // Build time.Time for filtering ts, err := time.ParseInLocation("2006-01-02 15:04", isoDate+" "+t, time.Local) if err != nil { continue } if ts.Before(now) || ts.After(max) { continue } out = append(out, map[string]any{ "id": m.MatchID, "home": m.Home, "away": m.Away, "competition": compName, "date": isoDate, "time": t, "venue": m.Venue, "home_logo_url": m.HomeLogoURL, "away_logo_url": m.AwayLogoURL, }) } } if len(out) == 0 { return false } _ = writeJSONAtomic(filepath.Join(cacheDir, "matches.json"), out) return true } // 2) Dynamic FACR endpoints based on saved settings settingsPath := filepath.Join(cacheDir, "settings.json") var clubID, clubType string if b, err := os.ReadFile(settingsPath); err == nil { var s struct { ClubID string `json:"club_id"` ClubType string `json:"club_type"` Youtube string `json:"youtube_url"` } if jsonErr := json.Unmarshal(b, &s); jsonErr == nil { clubID, clubType = strings.TrimSpace(s.ClubID), strings.TrimSpace(s.ClubType) // Opportunistically refresh YouTube if settings present and cache is older than ~23h if strings.TrimSpace(s.Youtube) != "" { maybeRefreshYouTubeCache(strings.TrimSpace(s.Youtube)) } } } if clubID != "" && clubType != "" { log.Printf("[prefetch] Fetching FACR data for club_id=%s club_type=%s", clubID, clubType) facrEndpoints := map[string]string{ fmt.Sprintf("/facr/club/%s/%s", clubType, clubID): "facr_club_info.json", fmt.Sprintf("/facr/club/%s/%s/table", clubType, clubID): "facr_tables.json", } for path, file := range facrEndpoints { url := baseURL + path out := filepath.Join(cacheDir, file) log.Printf("[prefetch] Fetching FACR: %s -> %s", url, out) if err := fetchToFile(client, url, out); err != nil { log.Printf("[prefetch] FACR ERROR: %s -> %s failed: %v", path, file, err) statuses = append(statuses, epStatus{Path: path, File: file, Ok: false, Error: err.Error()}) } else { log.Printf("[prefetch] FACR SUCCESS: updated %s", file) // Post-process logo URLs only when REMBG is enabled if config.AppConfig != nil && config.AppConfig.RembgEnabled { // Post-process club info logos to ensure transparent backgrounds on FACR logos if file == "facr_club_info.json" { if err := postProcessFACRClubInfoLogos(cacheDir); err != nil { log.Printf("[prefetch] FACR post-process WARN: %v", err) } } // Post-process tables logos similarly (team_logo_url) if file == "facr_tables.json" { if err := postProcessFACRTablesLogos(cacheDir); err != nil { log.Printf("[prefetch] FACR tables post-process WARN: %v", err) } } } else { log.Printf("[prefetch] REMBG disabled, skipping FACR logo post-processing for %s", file) } statuses = append(statuses, epStatus{Path: path, File: file, Ok: true}) } } // Try to build matches.json from freshly fetched FACR prefetch file if b, err := os.ReadFile(filepath.Join(cacheDir, "facr_club_info.json")); err == nil { if buildMatchesFromFACR(b) { createdMatches = true } } } else { log.Printf("[prefetch] WARNING: FACR skipped: missing club_id=%q or club_type=%q in settings", clubID, clubType) } // If FACR prefetch not available, try internal FACR cache as fallback if !createdMatches && clubID != "" && clubType != "" { facrCachePath := filepath.Join("cache", "facr", fmt.Sprintf("%s_%s_info.json", clubType, clubID)) if b, err := os.ReadFile(facrCachePath); err == nil { var cached struct { Data []byte `json:"data"` } if jsonErr := json.Unmarshal(b, &cached); jsonErr == nil && len(cached.Data) > 0 { if buildMatchesFromFACR(cached.Data) { createdMatches = true } } } } // Final fallback: copy events_upcoming.json or write empty list if !createdMatches { if b, err := os.ReadFile(filepath.Join(cacheDir, "events_upcoming.json")); err == nil { _ = os.WriteFile(filepath.Join(cacheDir, "matches.json"), b, 0o644) } else { _ = os.WriteFile(filepath.Join(cacheDir, "matches.json"), []byte("[]"), 0o644) } } // Meta timestamp _ = os.WriteFile(filepath.Join(cacheDir, "meta.json"), []byte(fmt.Sprintf(`{"lastUpdated":"%s"}`, time.Now().Format(time.RFC3339))), 0o644) // Detailed status for admin/debugging statusPayload := map[string]any{ "lastUpdated": time.Now().Format(time.RFC3339), "duration_ms": time.Since(start).Milliseconds(), "baseURL": baseURL, "endpoints": statuses, } _ = writeJSONAtomic(filepath.Join(cacheDir, "prefetch_status.json"), statusPayload) log.Printf("[prefetch] cycle finished in %s", time.Since(start)) } // in-flight guard to avoid overlapping cycles var prefetchInFlight int32 var prefetchPending int32 func doPrefetchCycleGuarded(client *http.Client, baseURL string) { if !atomic.CompareAndSwapInt32(&prefetchInFlight, 0, 1) { // Mark a rerun so we don't lose triggers (e.g., from setup) while a cycle is running atomic.StoreInt32(&prefetchPending, 1) log.Printf("[prefetch] in-flight: marked pending rerun") return } defer func() { atomic.StoreInt32(&prefetchInFlight, 0) // If a trigger arrived during this run, execute one more cycle immediately if atomic.SwapInt32(&prefetchPending, 0) == 1 { // Small delay to allow DB/cache writes to settle time.Sleep(500 * time.Millisecond) doPrefetchCycleGuarded(client, baseURL) } }() doPrefetchCycle(client, baseURL) } func isDuringMatch(cacheDir string) bool { // Helper: parse various time formats now := time.Now() parseDT := func(s string) (time.Time, bool) { s = strings.TrimSpace(s) if s == "" { return time.Time{}, false } layouts := []string{ "02.01.2006 15:04", time.RFC3339, "2006-01-02 15:04", } for _, layout := range layouts { if t, err := time.ParseInLocation(layout, s, time.Local); err == nil { return t, true } } return time.Time{}, false } within := func(ts time.Time) bool { start := ts.Add(-15 * time.Minute) // warmup window end := ts.Add(150 * time.Minute) // 2h30m incl. potential extra return now.After(start) && now.Before(end) } // Shared checker against a JSON blob that may contain competitions/matches checkBlob := func(b []byte) bool { if len(b) == 0 { return false } var payload struct { Competitions []struct { Matches []struct { DateTime string `json:"date_time"` Date string `json:"date"` Time string `json:"time"` Kickoff string `json:"kickoff"` } `json:"matches"` } `json:"competitions"` // Some alt shapes could be present; try a flat matches list too Matches []struct { DateTime string `json:"date_time"` Date string `json:"date"` Time string `json:"time"` Kickoff string `json:"kickoff"` } `json:"matches"` } if err := json.Unmarshal(b, &payload); err != nil { return false } // 1) competitions[*].matches for _, c := range payload.Competitions { for _, m := range c.Matches { var ts time.Time var ok bool if m.DateTime != "" { ts, ok = parseDT(m.DateTime) } else if m.Date != "" || m.Time != "" { ts, ok = parseDT(strings.TrimSpace(m.Date + " " + m.Time)) } else if m.Kickoff != "" { ts, ok = parseDT(m.Kickoff) } if ok && within(ts) { return true } } } // 2) flat matches for _, m := range payload.Matches { var ts time.Time var ok bool if m.DateTime != "" { ts, ok = parseDT(m.DateTime) } else if m.Date != "" || m.Time != "" { ts, ok = parseDT(strings.TrimSpace(m.Date + " " + m.Time)) } else if m.Kickoff != "" { ts, ok = parseDT(m.Kickoff) } if ok && within(ts) { return true } } return false } // Check all relevant cached files produced by prefetch files := []string{ filepath.Join(cacheDir, "facr_club_info.json"), filepath.Join(cacheDir, "facr_tables.json"), filepath.Join(cacheDir, "events_upcoming.json"), } for _, p := range files { if b, err := os.ReadFile(p); err == nil { if checkBlob(b) { return true } } } return false } // nextRandomDailyTime returns a time within the next ~24-36 hours at a random hour/minute // to avoid fixed schedule patterns. It picks a random hour in [0,23] on the next day // plus a small random minute/second jitter. func nextRandomDailyTime() time.Time { now := time.Now() // choose a window roughly next day nextDay := now.Add(24 * time.Hour) h := rand.Intn(24) m := rand.Intn(60) s := rand.Intn(60) t := time.Date(nextDay.Year(), nextDay.Month(), nextDay.Day(), h, m, s, 0, nextDay.Location()) // Add extra 0-3h random delay with 50% chance to introduce 24-27h spacing occasionally if rand.Intn(2) == 0 { t = t.Add(time.Duration(rand.Intn(180)) * time.Minute) } return t } // maybeRefreshYouTubeCache refreshes YouTube cache if the existing cache is older than ~23 hours. func maybeRefreshYouTubeCache(channelURL string) { cacheDir := filepath.Join("cache", "prefetch") cachePath := filepath.Join(cacheDir, "youtube_channel.json") info, err := os.Stat(cachePath) if err == nil { if time.Since(info.ModTime()) < 23*time.Hour { return // fresh enough } } _ = fetchYouTubeChannel(channelURL) } // fetchYouTubeChannelOnce reads youtube_url from cached settings and updates the cache. func fetchYouTubeChannelOnce() error { cacheDir := filepath.Join("cache", "prefetch") b, err := os.ReadFile(filepath.Join(cacheDir, "settings.json")) if err != nil { return fmt.Errorf("settings cache missing: %w", err) } var s struct { Youtube string `json:"youtube_url"` } if err := json.Unmarshal(b, &s); err != nil { return fmt.Errorf("invalid settings cache: %w", err) } ch := strings.TrimSpace(s.Youtube) if ch == "" { return fmt.Errorf("youtube_url not configured") } return fetchYouTubeChannel(ch) } // fetchYouTubeChannel downloads channel videos JSON from the external API and stores it on disk. func fetchYouTubeChannel(channel string) error { cacheDir := filepath.Join("cache", "prefetch") if err := os.MkdirAll(cacheDir, 0o755); err != nil { log.Printf("[prefetch] ERROR: Failed to create cache dir for YouTube: %v", err) return err } apiBase := "https://youtube.tdvorak.dev/channel_videos?channel=" u := apiBase + url.QueryEscape(strings.TrimSpace(channel)) log.Printf("[prefetch] Fetching YouTube channel: %s", u) client := &http.Client{Timeout: 25 * time.Second} req, err := http.NewRequest("GET", u, nil) if err != nil { return err } req.Header.Set("User-Agent", "fotbal-club-prefetch/1.0") resp, err := client.Do(req) if err != nil { log.Printf("[prefetch] YouTube ERROR: HTTP request failed: %v", err) return err } defer resp.Body.Close() if resp.StatusCode < 200 || resp.StatusCode >= 300 { log.Printf("[prefetch] YouTube ERROR: API returned status %d", resp.StatusCode) return fmt.Errorf("youtube api status %d", resp.StatusCode) } log.Printf("[prefetch] YouTube: Received %d response, saving to cache...", resp.StatusCode) outPath := filepath.Join(cacheDir, "youtube_channel.json") tmp := outPath + ".tmp" f, err := os.Create(tmp) if err != nil { return err } if _, err := io.Copy(f, resp.Body); err != nil { _ = f.Close() _ = os.Remove(tmp) return err } if err := f.Close(); err != nil { _ = os.Remove(tmp) return err } if err := os.Rename(tmp, outPath); err != nil { return err } // Save minimal header/meta for reference meta := map[string]string{ "fetched_at": time.Now().Format(time.RFC3339), "source": u, } _ = os.WriteFile(outPath+".hdr", func() []byte { b, _ := json.Marshal(meta); return b }(), 0o644) log.Printf("[prefetch] updated youtube_channel.json") return nil } // RegenerateFlatGalleryFiles creates flat photo lists from albums for frontend consumption. // This is a public wrapper that can be called from controllers. func RegenerateFlatGalleryFiles() error { cacheDir := filepath.Join("cache", "prefetch") albumsFile := filepath.Join(cacheDir, "zonerama_albums.json") // Read albums data, err := os.ReadFile(albumsFile) if err != nil { // If albums file doesn't exist yet, create empty flat files if os.IsNotExist(err) { emptyList := []interface{}{} emptyJSON, _ := json.MarshalIndent(emptyList, "", " ") _ = os.WriteFile(filepath.Join(cacheDir, "gallery.json"), emptyJSON, 0644) _ = os.WriteFile(filepath.Join(cacheDir, "zonerama_flat.json"), emptyJSON, 0644) hdr := map[string]string{ "fetched_at": time.Now().Format(time.RFC3339), "link": "", } hdrJSON, _ := json.MarshalIndent(hdr, "", " ") _ = os.WriteFile(filepath.Join(cacheDir, "zonerama_flat.json.hdr"), hdrJSON, 0644) return nil } return fmt.Errorf("failed to read albums: %w", err) } // Define album and photo structures type ZoneramaPhoto struct { ID string `json:"id"` PageURL string `json:"page_url"` Image1500 string `json:"image_1500"` } type ZoneramaAlbum struct { ID string `json:"id"` Title string `json:"title"` URL string `json:"url"` Photos []ZoneramaPhoto `json:"photos"` FetchedAt string `json:"fetched_at,omitempty"` } var albums []ZoneramaAlbum if err := json.Unmarshal(data, &albums); err != nil { return fmt.Errorf("failed to parse albums: %w", err) } // Flatten all photos from all albums type FlatPhoto struct { ID string `json:"id"` AlbumID string `json:"album_id"` PageURL string `json:"page_url"` Local string `json:"local"` Src string `json:"src"` Title string `json:"title"` } var flatPhotos []FlatPhoto var latestFetchTime time.Time var sourceLink string for _, album := range albums { // Track the most recent fetch time if album.FetchedAt != "" { if t, err := time.Parse(time.RFC3339, album.FetchedAt); err == nil { if t.After(latestFetchTime) { latestFetchTime = t sourceLink = album.URL } } } for _, photo := range album.Photos { flatPhotos = append(flatPhotos, FlatPhoto{ ID: photo.ID, AlbumID: album.ID, PageURL: photo.PageURL, Local: "", // Photos are not downloaded locally, served via proxy Src: photo.Image1500, Title: album.Title, // Use album title as photo title }) } } // Write gallery.json galleryJSON, err := json.MarshalIndent(flatPhotos, "", " ") if err != nil { return fmt.Errorf("failed to marshal gallery.json: %w", err) } if err := os.WriteFile(filepath.Join(cacheDir, "gallery.json"), galleryJSON, 0644); err != nil { return fmt.Errorf("failed to write gallery.json: %w", err) } // Write zonerama_flat.json (same content for compatibility) if err := os.WriteFile(filepath.Join(cacheDir, "zonerama_flat.json"), galleryJSON, 0644); err != nil { return fmt.Errorf("failed to write zonerama_flat.json: %w", err) } // Write header file with metadata fetchedAt := latestFetchTime.Format(time.RFC3339) if fetchedAt == "0001-01-01T00:00:00Z" { fetchedAt = time.Now().Format(time.RFC3339) } hdr := map[string]string{ "fetched_at": fetchedAt, "link": sourceLink, } hdrJSON, err := json.MarshalIndent(hdr, "", " ") if err != nil { return fmt.Errorf("failed to marshal header: %w", err) } if err := os.WriteFile(filepath.Join(cacheDir, "zonerama_flat.json.hdr"), hdrJSON, 0644); err != nil { return fmt.Errorf("failed to write header: %w", err) } log.Printf("[gallery] Regenerated flat gallery files: %d photos from %d albums", len(flatPhotos), len(albums)) return nil } func fetchToFile(client *http.Client, url string, outPath string) error { // Conditional GET support via sidecar header file hdrPath := outPath + ".hdr" var etag, lastMod string if b, err := os.ReadFile(hdrPath); err == nil { var m map[string]string if jsonErr := json.Unmarshal(b, &m); jsonErr == nil { etag = m["etag"] lastMod = m["last_modified"] } } req, err := http.NewRequest("GET", url, nil) if err != nil { return err } // Identify ourselves req.Header.Set("User-Agent", "fotbal-club-prefetch/1.0") if etag != "" { req.Header.Set("If-None-Match", etag) } if lastMod != "" { req.Header.Set("If-Modified-Since", lastMod) } resp, err := client.Do(req) if err != nil { return err } defer resp.Body.Close() if resp.StatusCode < 200 || resp.StatusCode >= 300 { if resp.StatusCode == http.StatusNotModified { // nothing to update return nil } return fmt.Errorf("unexpected status %d", resp.StatusCode) } // Write to temp and then atomically replace to avoid partial writes if err := os.MkdirAll(filepath.Dir(outPath), 0o755); err != nil { return err } tmp := outPath + ".tmp" f, err := os.Create(tmp) if err != nil { return err } _, copyErr := io.Copy(f, resp.Body) closeErr := f.Close() if copyErr != nil { _ = os.Remove(tmp) return copyErr } if closeErr != nil { _ = os.Remove(tmp) return closeErr } if err := os.Rename(tmp, outPath); err != nil { return err } // Persist ETag/Last-Modified for next cycle meta := map[string]string{ "etag": resp.Header.Get("ETag"), "last_modified": resp.Header.Get("Last-Modified"), "fetched_at": time.Now().Format(time.RFC3339), } if b, err := json.Marshal(meta); err == nil { _ = os.WriteFile(hdrPath, b, 0o644) } return nil } // postProcessFACRClubInfoLogos rewrites home_logo_url/away_logo_url in facr_club_info.json // to point to locally cached transparent PNGs produced by rembg. // Best-effort: logs warning on failure and preserves original file. func postProcessFACRClubInfoLogos(cacheDir string) error { p := filepath.Join(cacheDir, "facr_club_info.json") b, err := os.ReadFile(p) if err != nil { return err } // Fast check: if file is tiny or empty, skip if len(strings.TrimSpace(string(b))) == 0 { return fmt.Errorf("facr_club_info.json empty") } // Parse into a minimal shape we care about var payload struct { Competitions []struct { Matches []struct { HomeLogoURL string `json:"home_logo_url"` AwayLogoURL string `json:"away_logo_url"` } `json:"matches"` } `json:"competitions"` } if err := json.Unmarshal(b, &payload); err != nil { return fmt.Errorf("parse facr_club_info: %w", err) } // Build a map of original -> processed URL to avoid duplicate processing seen := make(map[string]string, 64) changed := false // Walk and rewrite for ci := range payload.Competitions { for mi := range payload.Competitions[ci].Matches { h := strings.TrimSpace(payload.Competitions[ci].Matches[mi].HomeLogoURL) if h != "" { if rep, ok := seen[h]; ok { if rep != h && rep != "" { payload.Competitions[ci].Matches[mi].HomeLogoURL = rep changed = true } } else { if newURL, err := ProcessFACRLogo(h); err == nil && newURL != "" && newURL != h { seen[h] = newURL payload.Competitions[ci].Matches[mi].HomeLogoURL = newURL changed = true } else { seen[h] = h } } } a := strings.TrimSpace(payload.Competitions[ci].Matches[mi].AwayLogoURL) if a != "" { if rep, ok := seen[a]; ok { if rep != a && rep != "" { payload.Competitions[ci].Matches[mi].AwayLogoURL = rep changed = true } } else { if newURL, err := ProcessFACRLogo(a); err == nil && newURL != "" && newURL != a { seen[a] = newURL payload.Competitions[ci].Matches[mi].AwayLogoURL = newURL changed = true } else { seen[a] = a } } } } } if !changed { return nil } // Merge back into original JSON to preserve other fields // Unmarshal original into generic map, then overlay changes var orig map[string]any if err := json.Unmarshal(b, &orig); err != nil { return fmt.Errorf("parse orig for merge: %w", err) } // Rebuild competitions with rewritten logos if comps, ok := orig["competitions"].([]any); ok { for i := range comps { comp, _ := comps[i].(map[string]any) if comp == nil { continue } if matches, ok2 := comp["matches"].([]any); ok2 { for j := range matches { m, _ := matches[j].(map[string]any) if m == nil { continue } // Safely get strings hv := "" av := "" if s, ok3 := m["home_logo_url"].(string); ok3 { hv = s } if s, ok3 := m["away_logo_url"].(string); ok3 { av = s } if rep, ok3 := seen[hv]; ok3 && rep != "" && rep != hv { m["home_logo_url"] = rep } if rep, ok3 := seen[av]; ok3 && rep != "" && rep != av { m["away_logo_url"] = rep } } } } } // Write back atomically if err := writeJSONAtomic(p, orig); err != nil { return err } return nil } // postProcessFACRTablesLogos rewrites team_logo_url entries in facr_tables.json // to point to locally cached transparent PNGs produced by rembg. // Best-effort: logs warning on failure and preserves original file. func postProcessFACRTablesLogos(cacheDir string) error { p := filepath.Join(cacheDir, "facr_tables.json") b, err := os.ReadFile(p) if err != nil { return err } // Fast check: if file is tiny or empty, skip if len(strings.TrimSpace(string(b))) == 0 { return fmt.Errorf("facr_tables.json empty") } // Parse minimal shape var payload struct { Competitions []struct { Table struct { Overall []struct { TeamLogoURL string `json:"team_logo_url"` } `json:"overall"` } `json:"table"` } `json:"competitions"` } if err := json.Unmarshal(b, &payload); err != nil { return fmt.Errorf("parse facr_tables: %w", err) } seen := make(map[string]string, 64) changed := false for ci := range payload.Competitions { rows := payload.Competitions[ci].Table.Overall for ri := range rows { s := strings.TrimSpace(rows[ri].TeamLogoURL) if s == "" { continue } if rep, ok := seen[s]; ok { if rep != "" && rep != s { payload.Competitions[ci].Table.Overall[ri].TeamLogoURL = rep changed = true } } else { if purl, err := ProcessFACRLogo(s); err == nil && strings.TrimSpace(purl) != "" && purl != s { seen[s] = purl payload.Competitions[ci].Table.Overall[ri].TeamLogoURL = purl changed = true } else { seen[s] = s } } } } if !changed { return nil } // Merge back into original JSON preserving other fields var orig map[string]any if err := json.Unmarshal(b, &orig); err != nil { return fmt.Errorf("parse orig for merge: %w", err) } if comps, ok := orig["competitions"].([]any); ok { for i := range comps { comp, _ := comps[i].(map[string]any) if comp == nil { continue } tbl, _ := comp["table"].(map[string]any) if tbl == nil { continue } overall, _ := tbl["overall"].([]any) for j := range overall { row, _ := overall[j].(map[string]any) if row == nil { continue } if s, ok3 := row["team_logo_url"].(string); ok3 { if rep, ok := seen[s]; ok && rep != "" && rep != s { row["team_logo_url"] = rep } } } } } if err := writeJSONAtomic(p, orig); err != nil { return err } return nil }