TDvorak/MyClub
1
0
Fork 0
mirror of https://github.com/Dvorinka/MyClubServer.git synced 2026-06-04 10:42:57 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

dev day #99

Browse Source
This commit is contained in:
Tomas Dvorak
2025-11-21 08:44:44 +01:00
parent c941313fd5
commit f5b6f83974
108 changed files with 8642 additions and 5871 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/services/prefetch_service.go
+385 -142
View File
@@ -14,6 +14,8 @@ import (
"strings"
"sync/atomic"
"time"
"fotbal-club/internal/config"
)
// StartPrefetcher starts a background job that periodically fetches
@@ -81,7 +83,7 @@ func fetchZonerama(link string) error {
// Increase timeout to 60s since the API can take longer to fetch
client := &http.Client{Timeout: 60 * time.Second}
// Retry logic with exponential backoff (3 attempts)
var resp *http.Response
var err error
@@ -114,7 +116,7 @@ func fetchZonerama(link string) error {
time.Sleep(backoff)
}
}
if err != nil {
return err
}
@@ -165,7 +167,7 @@ func fetchZonerama(link string) error {
}
log.Printf("[prefetch] Zonerama: Fetching %d albums with photos...", len(profile.Albums))
// Fetch individual albums with photos
photoLimit := envInt("ZONERAMA_PHOTO_LIMIT", 50) // Default 50 photos per album
fetchedAlbums, err := fetchZoneramaAlbums(profile.Albums, photoLimit, client)
@@ -223,7 +225,7 @@ func fetchZoneramaAlbums(albums []struct {
// Fetch album with photos
apiURL := fmt.Sprintf("https://zonerama.tdvorak.dev/zonerama-album?link=%s&photo_limit=%d",
url.QueryEscape(album.URL), photoLimit)
log.Printf("[prefetch] Zonerama: Fetching album %d/%d: %s", i+1, len(albums), album.URL)
req, err := http.NewRequest("GET", apiURL, nil)
@@ -238,7 +240,7 @@ func fetchZoneramaAlbums(albums []struct {
log.Printf("[prefetch] Zonerama: Failed to fetch album %s: %v", album.ID, err)
continue
}
if resp.StatusCode != 200 {
log.Printf("[prefetch] Zonerama: Album %s returned status %d", album.ID, resp.StatusCode)
_ = resp.Body.Close()
@@ -255,9 +257,9 @@ func fetchZoneramaAlbums(albums []struct {
albumData.FetchedAt = fetchedAt
result = append(result, albumData)
log.Printf("[prefetch] Zonerama: Album %s fetched with %d photos", albumData.ID, len(albumData.Photos))
// Small delay between requests to avoid overwhelming the API
if i < len(albums)-1 {
time.Sleep(500 * time.Millisecond)
@@ -506,28 +508,28 @@ func PrefetchOnce(baseURL string) {
// doPrefetchCycle performs one full prefetch cycle: static public endpoints + dynamic FACR endpoints.
func doPrefetchCycle(client *http.Client, baseURL string) {
cacheDir := filepath.Join("cache", "prefetch")
_ = os.MkdirAll(cacheDir, 0o755)
start := time.Now()
// Track endpoint fetch statuses for admin/debugging payload
type epStatus struct {
Path string `json:"path"`
File string `json:"file"`
Ok bool `json:"ok"`
Error string `json:"error,omitempty"`
}
var statuses []epStatus
cacheDir := filepath.Join("cache", "prefetch")
_ = os.MkdirAll(cacheDir, 0o755)
start := time.Now()
// Track endpoint fetch statuses for admin/debugging payload
type epStatus struct {
Path string `json:"path"`
File string `json:"file"`
Ok bool `json:"ok"`
Error string `json:"error,omitempty"`
}
var statuses []epStatus
// 1) Static public endpoints
endpoints := map[string]string{
"/settings": "settings.json",
"/seo": "seo.json",
"/articles?page=1&page_size=10&published=true": "articles.json",
"/sponsors": "sponsors.json",
"/events/upcoming": "events_upcoming.json",
"/public/team-logo-overrides": "team_logo_overrides.json",
"/competition-aliases": "competition_aliases.json",
}
// 1) Static public endpoints
endpoints := map[string]string{
"/settings": "settings.json",
"/seo": "seo.json",
"/articles?page=1&page_size=10&published=true": "articles.json",
"/sponsors": "sponsors.json",
"/events/upcoming": "events_upcoming.json",
"/public/team-logo-overrides": "team_logo_overrides.json",
"/competition-aliases": "competition_aliases.json",
}
statuses = make([]epStatus, 0, len(endpoints))
for path, file := range endpoints {
url := baseURL + path
@@ -542,82 +544,96 @@ func doPrefetchCycle(client *http.Client, baseURL string) {
}
}
// Prepare matches.json from FACR data if available; fallback to events_upcoming.json
createdMatches := false
buildMatchesFromFACR := func(data []byte) bool {
type facrMatch struct {
DateTime string `json:"date_time"`
Home string `json:"home"`
Away string `json:"away"`
Venue string `json:"venue"`
HomeLogoURL string `json:"home_logo_url"`
AwayLogoURL string `json:"away_logo_url"`
MatchID string `json:"match_id"`
}
var facr struct {
Competitions []struct {
Name string `json:"name"`
Code string `json:"code"`
Matches []facrMatch `json:"matches"`
} `json:"competitions"`
}
if err := json.Unmarshal(data, &facr); err != nil {
return false
}
// Collect upcoming matches (next 30 days) in simplified format
now := time.Now()
max := now.Add(30 * 24 * time.Hour)
var out []map[string]any
for _, c := range facr.Competitions {
compName := strings.TrimSpace(c.Name)
if compName == "" {
compName = strings.TrimSpace(c.Code)
}
for _, m := range c.Matches {
dt := strings.TrimSpace(m.DateTime)
if dt == "" {
continue
}
// dt like "12.08.2023 18:00"
parts := strings.SplitN(dt, " ", 2)
d := parts[0]
t := ""
if len(parts) > 1 { t = parts[1] }
// parse date dd.mm.yyyy
dd := strings.Split(d, ".")
if len(dd) < 3 { continue }
day := dd[0]
month := dd[1]
year := dd[2]
if len(month) == 1 { month = "0" + month }
if len(day) == 1 { day = "0" + day }
isoDate := year + "-" + month + "-" + day
if len(t) >= 5 {
t = t[:5]
} else {
t = "18:00"
}
// Build time.Time for filtering
ts, err := time.ParseInLocation("2006-01-02 15:04", isoDate+" "+t, time.Local)
if err != nil { continue }
if ts.Before(now) || ts.After(max) { continue }
out = append(out, map[string]any{
"id": m.MatchID,
"home": m.Home,
"away": m.Away,
"competition": compName,
"date": isoDate,
"time": t,
"venue": m.Venue,
"home_logo_url": m.HomeLogoURL,
"away_logo_url": m.AwayLogoURL,
})
}
}
if len(out) == 0 { return false }
_ = writeJSONAtomic(filepath.Join(cacheDir, "matches.json"), out)
return true
}
// Prepare matches.json from FACR data if available; fallback to events_upcoming.json
createdMatches := false
buildMatchesFromFACR := func(data []byte) bool {
type facrMatch struct {
DateTime string `json:"date_time"`
Home string `json:"home"`
Away string `json:"away"`
Venue string `json:"venue"`
HomeLogoURL string `json:"home_logo_url"`
AwayLogoURL string `json:"away_logo_url"`
MatchID string `json:"match_id"`
}
var facr struct {
Competitions []struct {
Name string `json:"name"`
Code string `json:"code"`
Matches []facrMatch `json:"matches"`
} `json:"competitions"`
}
if err := json.Unmarshal(data, &facr); err != nil {
return false
}
// Collect upcoming matches (next 30 days) in simplified format
now := time.Now()
max := now.Add(30 * 24 * time.Hour)
var out []map[string]any
for _, c := range facr.Competitions {
compName := strings.TrimSpace(c.Name)
if compName == "" {
compName = strings.TrimSpace(c.Code)
}
for _, m := range c.Matches {
dt := strings.TrimSpace(m.DateTime)
if dt == "" {
continue
}
// dt like "12.08.2023 18:00"
parts := strings.SplitN(dt, " ", 2)
d := parts[0]
t := ""
if len(parts) > 1 {
t = parts[1]
}
// parse date dd.mm.yyyy
dd := strings.Split(d, ".")
if len(dd) < 3 {
continue
}
day := dd[0]
month := dd[1]
year := dd[2]
if len(month) == 1 {
month = "0" + month
}
if len(day) == 1 {
day = "0" + day
}
isoDate := year + "-" + month + "-" + day
if len(t) >= 5 {
t = t[:5]
} else {
t = "18:00"
}
// Build time.Time for filtering
ts, err := time.ParseInLocation("2006-01-02 15:04", isoDate+" "+t, time.Local)
if err != nil {
continue
}
if ts.Before(now) || ts.After(max) {
continue
}
out = append(out, map[string]any{
"id": m.MatchID,
"home": m.Home,
"away": m.Away,
"competition": compName,
"date": isoDate,
"time": t,
"venue": m.Venue,
"home_logo_url": m.HomeLogoURL,
"away_logo_url": m.AwayLogoURL,
})
}
}
if len(out) == 0 {
return false
}
_ = writeJSONAtomic(filepath.Join(cacheDir, "matches.json"), out)
return true
}
// 2) Dynamic FACR endpoints based on saved settings
settingsPath := filepath.Join(cacheDir, "settings.json")
@@ -651,12 +667,31 @@ func doPrefetchCycle(client *http.Client, baseURL string) {
statuses = append(statuses, epStatus{Path: path, File: file, Ok: false, Error: err.Error()})
} else {
log.Printf("[prefetch] FACR SUCCESS: updated %s", file)
// Post-process logo URLs only when REMBG is enabled
if config.AppConfig != nil && config.AppConfig.RembgEnabled {
// Post-process club info logos to ensure transparent backgrounds on FACR logos
if file == "facr_club_info.json" {
if err := postProcessFACRClubInfoLogos(cacheDir); err != nil {
log.Printf("[prefetch] FACR post-process WARN: %v", err)
}
}
// Post-process tables logos similarly (team_logo_url)
if file == "facr_tables.json" {
if err := postProcessFACRTablesLogos(cacheDir); err != nil {
log.Printf("[prefetch] FACR tables post-process WARN: %v", err)
}
}
} else {
log.Printf("[prefetch] REMBG disabled, skipping FACR logo post-processing for %s", file)
}
statuses = append(statuses, epStatus{Path: path, File: file, Ok: true})
}
}
// Try to build matches.json from freshly fetched FACR prefetch file
if b, err := os.ReadFile(filepath.Join(cacheDir, "facr_club_info.json")); err == nil {
if buildMatchesFromFACR(b) { createdMatches = true }
if buildMatchesFromFACR(b) {
createdMatches = true
}
}
} else {
log.Printf("[prefetch] WARNING: FACR skipped: missing club_id=%q or club_type=%q in settings", clubID, clubType)
@@ -666,9 +701,13 @@ func doPrefetchCycle(client *http.Client, baseURL string) {
if !createdMatches && clubID != "" && clubType != "" {
facrCachePath := filepath.Join("cache", "facr", fmt.Sprintf("%s_%s_info.json", clubType, clubID))
if b, err := os.ReadFile(facrCachePath); err == nil {
var cached struct{ Data []byte `json:"data"` }
var cached struct {
Data []byte `json:"data"`
}
if jsonErr := json.Unmarshal(b, &cached); jsonErr == nil && len(cached.Data) > 0 {
if buildMatchesFromFACR(cached.Data) { createdMatches = true }
if buildMatchesFromFACR(cached.Data) {
createdMatches = true
}
}
}
}
@@ -700,22 +739,22 @@ var prefetchInFlight int32
var prefetchPending int32
func doPrefetchCycleGuarded(client *http.Client, baseURL string) {
if !atomic.CompareAndSwapInt32(&prefetchInFlight, 0, 1) {
// Mark a rerun so we don't lose triggers (e.g., from setup) while a cycle is running
atomic.StoreInt32(&prefetchPending, 1)
log.Printf("[prefetch] in-flight: marked pending rerun")
return
}
defer func() {
atomic.StoreInt32(&prefetchInFlight, 0)
// If a trigger arrived during this run, execute one more cycle immediately
if atomic.SwapInt32(&prefetchPending, 0) == 1 {
// Small delay to allow DB/cache writes to settle
time.Sleep(500 * time.Millisecond)
doPrefetchCycleGuarded(client, baseURL)
}
}()
doPrefetchCycle(client, baseURL)
if !atomic.CompareAndSwapInt32(&prefetchInFlight, 0, 1) {
// Mark a rerun so we don't lose triggers (e.g., from setup) while a cycle is running
atomic.StoreInt32(&prefetchPending, 1)
log.Printf("[prefetch] in-flight: marked pending rerun")
return
}
defer func() {
atomic.StoreInt32(&prefetchInFlight, 0)
// If a trigger arrived during this run, execute one more cycle immediately
if atomic.SwapInt32(&prefetchPending, 0) == 1 {
// Small delay to allow DB/cache writes to settle
time.Sleep(500 * time.Millisecond)
doPrefetchCycleGuarded(client, baseURL)
}
}()
doPrefetchCycle(client, baseURL)
}
func isDuringMatch(cacheDir string) bool {
@@ -935,7 +974,7 @@ func fetchYouTubeChannel(channel string) error {
func RegenerateFlatGalleryFiles() error {
cacheDir := filepath.Join("cache", "prefetch")
albumsFile := filepath.Join(cacheDir, "zonerama_albums.json")
// Read albums
data, err := os.ReadFile(albumsFile)
if err != nil {
@@ -945,7 +984,7 @@ func RegenerateFlatGalleryFiles() error {
emptyJSON, _ := json.MarshalIndent(emptyList, "", " ")
_ = os.WriteFile(filepath.Join(cacheDir, "gallery.json"), emptyJSON, 0644)
_ = os.WriteFile(filepath.Join(cacheDir, "zonerama_flat.json"), emptyJSON, 0644)
hdr := map[string]string{
"fetched_at": time.Now().Format(time.RFC3339),
"link": "",
@@ -956,7 +995,7 @@ func RegenerateFlatGalleryFiles() error {
}
return fmt.Errorf("failed to read albums: %w", err)
}
// Define album and photo structures
type ZoneramaPhoto struct {
ID string `json:"id"`
@@ -970,26 +1009,26 @@ func RegenerateFlatGalleryFiles() error {
Photos []ZoneramaPhoto `json:"photos"`
FetchedAt string `json:"fetched_at,omitempty"`
}
var albums []ZoneramaAlbum
if err := json.Unmarshal(data, &albums); err != nil {
return fmt.Errorf("failed to parse albums: %w", err)
}
// Flatten all photos from all albums
type FlatPhoto struct {
ID string `json:"id"`
AlbumID string `json:"album_id"`
PageURL string `json:"page_url"`
Local string `json:"local"`
Src string `json:"src"`
Title string `json:"title"`
ID string `json:"id"`
AlbumID string `json:"album_id"`
PageURL string `json:"page_url"`
Local string `json:"local"`
Src string `json:"src"`
Title string `json:"title"`
}
var flatPhotos []FlatPhoto
var latestFetchTime time.Time
var sourceLink string
for _, album := range albums {
// Track the most recent fetch time
if album.FetchedAt != "" {
@@ -1000,7 +1039,7 @@ func RegenerateFlatGalleryFiles() error {
}
}
}
for _, photo := range album.Photos {
flatPhotos = append(flatPhotos, FlatPhoto{
ID: photo.ID,
@@ -1012,7 +1051,7 @@ func RegenerateFlatGalleryFiles() error {
})
}
}
// Write gallery.json
galleryJSON, err := json.MarshalIndent(flatPhotos, "", " ")
if err != nil {
@@ -1021,18 +1060,18 @@ func RegenerateFlatGalleryFiles() error {
if err := os.WriteFile(filepath.Join(cacheDir, "gallery.json"), galleryJSON, 0644); err != nil {
return fmt.Errorf("failed to write gallery.json: %w", err)
}
// Write zonerama_flat.json (same content for compatibility)
if err := os.WriteFile(filepath.Join(cacheDir, "zonerama_flat.json"), galleryJSON, 0644); err != nil {
return fmt.Errorf("failed to write zonerama_flat.json: %w", err)
}
// Write header file with metadata
fetchedAt := latestFetchTime.Format(time.RFC3339)
if fetchedAt == "0001-01-01T00:00:00Z" {
fetchedAt = time.Now().Format(time.RFC3339)
}
hdr := map[string]string{
"fetched_at": fetchedAt,
"link": sourceLink,
@@ -1044,7 +1083,7 @@ func RegenerateFlatGalleryFiles() error {
if err := os.WriteFile(filepath.Join(cacheDir, "zonerama_flat.json.hdr"), hdrJSON, 0644); err != nil {
return fmt.Errorf("failed to write header: %w", err)
}
log.Printf("[gallery] Regenerated flat gallery files: %d photos from %d albums", len(flatPhotos), len(albums))
return nil
}
@@ -1120,3 +1159,207 @@ func fetchToFile(client *http.Client, url string, outPath string) error {
}
return nil
}
// postProcessFACRClubInfoLogos rewrites home_logo_url/away_logo_url in facr_club_info.json
// to point to locally cached transparent PNGs produced by rembg.
// Best-effort: logs warning on failure and preserves original file.
func postProcessFACRClubInfoLogos(cacheDir string) error {
p := filepath.Join(cacheDir, "facr_club_info.json")
b, err := os.ReadFile(p)
if err != nil {
return err
}
// Fast check: if file is tiny or empty, skip
if len(strings.TrimSpace(string(b))) == 0 {
return fmt.Errorf("facr_club_info.json empty")
}
// Parse into a minimal shape we care about
var payload struct {
Competitions []struct {
Matches []struct {
HomeLogoURL string `json:"home_logo_url"`
AwayLogoURL string `json:"away_logo_url"`
} `json:"matches"`
} `json:"competitions"`
}
if err := json.Unmarshal(b, &payload); err != nil {
return fmt.Errorf("parse facr_club_info: %w", err)
}
// Build a map of original -> processed URL to avoid duplicate processing
seen := make(map[string]string, 64)
changed := false
// Walk and rewrite
for ci := range payload.Competitions {
for mi := range payload.Competitions[ci].Matches {
h := strings.TrimSpace(payload.Competitions[ci].Matches[mi].HomeLogoURL)
if h != "" {
if rep, ok := seen[h]; ok {
if rep != h && rep != "" {
payload.Competitions[ci].Matches[mi].HomeLogoURL = rep
changed = true
}
} else {
if newURL, err := ProcessFACRLogo(h); err == nil && newURL != "" && newURL != h {
seen[h] = newURL
payload.Competitions[ci].Matches[mi].HomeLogoURL = newURL
changed = true
} else {
seen[h] = h
}
}
}
a := strings.TrimSpace(payload.Competitions[ci].Matches[mi].AwayLogoURL)
if a != "" {
if rep, ok := seen[a]; ok {
if rep != a && rep != "" {
payload.Competitions[ci].Matches[mi].AwayLogoURL = rep
changed = true
}
} else {
if newURL, err := ProcessFACRLogo(a); err == nil && newURL != "" && newURL != a {
seen[a] = newURL
payload.Competitions[ci].Matches[mi].AwayLogoURL = newURL
changed = true
} else {
seen[a] = a
}
}
}
}
}
if !changed {
return nil
}
// Merge back into original JSON to preserve other fields
// Unmarshal original into generic map, then overlay changes
var orig map[string]any
if err := json.Unmarshal(b, &orig); err != nil {
return fmt.Errorf("parse orig for merge: %w", err)
}
// Rebuild competitions with rewritten logos
if comps, ok := orig["competitions"].([]any); ok {
for i := range comps {
comp, _ := comps[i].(map[string]any)
if comp == nil {
continue
}
if matches, ok2 := comp["matches"].([]any); ok2 {
for j := range matches {
m, _ := matches[j].(map[string]any)
if m == nil {
continue
}
// Safely get strings
hv := ""
av := ""
if s, ok3 := m["home_logo_url"].(string); ok3 {
hv = s
}
if s, ok3 := m["away_logo_url"].(string); ok3 {
av = s
}
if rep, ok3 := seen[hv]; ok3 && rep != "" && rep != hv {
m["home_logo_url"] = rep
}
if rep, ok3 := seen[av]; ok3 && rep != "" && rep != av {
m["away_logo_url"] = rep
}
}
}
}
}
// Write back atomically
if err := writeJSONAtomic(p, orig); err != nil {
return err
}
return nil
}
// postProcessFACRTablesLogos rewrites team_logo_url entries in facr_tables.json
// to point to locally cached transparent PNGs produced by rembg.
// Best-effort: logs warning on failure and preserves original file.
func postProcessFACRTablesLogos(cacheDir string) error {
p := filepath.Join(cacheDir, "facr_tables.json")
b, err := os.ReadFile(p)
if err != nil {
return err
}
// Fast check: if file is tiny or empty, skip
if len(strings.TrimSpace(string(b))) == 0 {
return fmt.Errorf("facr_tables.json empty")
}
// Parse minimal shape
var payload struct {
Competitions []struct {
Table struct {
Overall []struct {
TeamLogoURL string `json:"team_logo_url"`
} `json:"overall"`
} `json:"table"`
} `json:"competitions"`
}
if err := json.Unmarshal(b, &payload); err != nil {
return fmt.Errorf("parse facr_tables: %w", err)
}
seen := make(map[string]string, 64)
changed := false
for ci := range payload.Competitions {
rows := payload.Competitions[ci].Table.Overall
for ri := range rows {
s := strings.TrimSpace(rows[ri].TeamLogoURL)
if s == "" {
continue
}
if rep, ok := seen[s]; ok {
if rep != "" && rep != s {
payload.Competitions[ci].Table.Overall[ri].TeamLogoURL = rep
changed = true
}
} else {
if purl, err := ProcessFACRLogo(s); err == nil && strings.TrimSpace(purl) != "" && purl != s {
seen[s] = purl
payload.Competitions[ci].Table.Overall[ri].TeamLogoURL = purl
changed = true
} else {
seen[s] = s
}
}
}
}
if !changed {
return nil
}
// Merge back into original JSON preserving other fields
var orig map[string]any
if err := json.Unmarshal(b, &orig); err != nil {
return fmt.Errorf("parse orig for merge: %w", err)
}
if comps, ok := orig["competitions"].([]any); ok {
for i := range comps {
comp, _ := comps[i].(map[string]any)
if comp == nil {
continue
}
tbl, _ := comp["table"].(map[string]any)
if tbl == nil {
continue
}
overall, _ := tbl["overall"].([]any)
for j := range overall {
row, _ := overall[j].(map[string]any)
if row == nil {
continue
}
if s, ok3 := row["team_logo_url"].(string); ok3 {
if rep, ok := seen[s]; ok && rep != "" && rep != s {
row["team_logo_url"] = rep
}
}
}
}
}
if err := writeJSONAtomic(p, orig); err != nil {
return err
}
return nil
}