package projectstate import ( "encoding/json" "fmt" "io/fs" "os" "path/filepath" "strings" "time" ) type SourceState struct { Name string `json:"name"` Type string `json:"type"` URL string `json:"url,omitempty"` Hash string `json:"hash,omitempty"` LastSync time.Time `json:"last_sync,omitempty"` DocCount int `json:"doc_count"` LastError string `json:"last_error,omitempty"` } type SourceStateFile struct { UpdatedAt time.Time `json:"updated_at"` Sources map[string]*SourceState `json:"sources"` } type DocsStats struct { DocumentCount int LastUpdated time.Time BySource map[string]int StorageBytes int64 } type docSummary struct { Source string `json:"source"` } const sourceStateFileName = "source_state.json" func LoadSourceState(metadataDir string) (*SourceStateFile, error) { path := filepath.Join(metadataDir, sourceStateFileName) b, err := os.ReadFile(path) if err != nil { if os.IsNotExist(err) { return &SourceStateFile{UpdatedAt: time.Now(), Sources: map[string]*SourceState{}}, nil } return nil, err } var state SourceStateFile if err := json.Unmarshal(b, &state); err != nil { return nil, err } if state.Sources == nil { state.Sources = map[string]*SourceState{} } return &state, nil } func SaveSourceState(metadataDir string, state *SourceStateFile) error { if state == nil { return fmt.Errorf("state is required") } if state.Sources == nil { state.Sources = map[string]*SourceState{} } state.UpdatedAt = time.Now() if err := os.MkdirAll(metadataDir, 0o755); err != nil { return err } path := filepath.Join(metadataDir, sourceStateFileName) b, err := json.MarshalIndent(state, "", " ") if err != nil { return err } return os.WriteFile(path, b, 0o644) } func CollectDocsStats(docsDir string) (*DocsStats, error) { stats := &DocsStats{BySource: map[string]int{}} err := filepath.WalkDir(docsDir, func(path string, d fs.DirEntry, err error) error { if err != nil { if os.IsNotExist(err) { return nil } return err } if d.IsDir() { return nil } info, infoErr := d.Info() if infoErr != nil { return infoErr } stats.StorageBytes += info.Size() if info.ModTime().After(stats.LastUpdated) { stats.LastUpdated = info.ModTime() } ext := strings.ToLower(filepath.Ext(path)) if ext != ".json" && ext != ".md" && ext != ".txt" { return nil } stats.DocumentCount++ if ext == ".json" { b, readErr := os.ReadFile(path) if readErr != nil { return nil } var d docSummary if err := json.Unmarshal(b, &d); err == nil { source := strings.TrimSpace(d.Source) if source != "" { stats.BySource[source]++ } } } return nil }) if err != nil { return nil, err } return stats, nil }