Devour/internal/projectstate/state.go

package projectstate

import (
	"encoding/json"
	"fmt"
	"io/fs"
	"os"
	"path/filepath"
	"strings"
	"time"
)

type SourceState struct {
	Name      string    `json:"name"`
	Type      string    `json:"type"`
	URL       string    `json:"url,omitempty"`
	Hash      string    `json:"hash,omitempty"`
	LastSync  time.Time `json:"last_sync,omitempty"`
	DocCount  int       `json:"doc_count"`
	LastError string    `json:"last_error,omitempty"`
}

type SourceStateFile struct {
	UpdatedAt time.Time               `json:"updated_at"`
	Sources   map[string]*SourceState `json:"sources"`
}

type DocsStats struct {
	DocumentCount int
	LastUpdated   time.Time
	BySource      map[string]int
	StorageBytes  int64
}

type docSummary struct {
	Source string `json:"source"`
}

const sourceStateFileName = "source_state.json"

func LoadSourceState(metadataDir string) (*SourceStateFile, error) {
	path := filepath.Join(metadataDir, sourceStateFileName)
	b, err := os.ReadFile(path)
	if err != nil {
		if os.IsNotExist(err) {
			return &SourceStateFile{UpdatedAt: time.Now(), Sources: map[string]*SourceState{}}, nil
		}
		return nil, err
	}

	var state SourceStateFile
	if err := json.Unmarshal(b, &state); err != nil {
		return nil, err
	}
	if state.Sources == nil {
		state.Sources = map[string]*SourceState{}
	}
	return &state, nil
}

func SaveSourceState(metadataDir string, state *SourceStateFile) error {
	if state == nil {
		return fmt.Errorf("state is required")
	}
	if state.Sources == nil {
		state.Sources = map[string]*SourceState{}
	}
	state.UpdatedAt = time.Now()

	if err := os.MkdirAll(metadataDir, 0o755); err != nil {
		return err
	}
	path := filepath.Join(metadataDir, sourceStateFileName)
	b, err := json.MarshalIndent(state, "", "  ")
	if err != nil {
		return err
	}
	return os.WriteFile(path, b, 0o644)
}

func CollectDocsStats(docsDir string) (*DocsStats, error) {
	stats := &DocsStats{BySource: map[string]int{}}

	err := filepath.WalkDir(docsDir, func(path string, d fs.DirEntry, err error) error {
		if err != nil {
			if os.IsNotExist(err) {
				return nil
			}
			return err
		}
		if d.IsDir() {
			return nil
		}

		info, infoErr := d.Info()
		if infoErr != nil {
			return infoErr
		}
		stats.StorageBytes += info.Size()
		if info.ModTime().After(stats.LastUpdated) {
			stats.LastUpdated = info.ModTime()
		}

		ext := strings.ToLower(filepath.Ext(path))
		if ext != ".json" && ext != ".md" && ext != ".txt" {
			return nil
		}
		stats.DocumentCount++

		if ext == ".json" {
			b, readErr := os.ReadFile(path)
			if readErr != nil {
				return nil
			}
			var d docSummary
			if err := json.Unmarshal(b, &d); err == nil {
				source := strings.TrimSpace(d.Source)
				if source != "" {
					stats.BySource[source]++
				}
			}
		}
		return nil
	})
	if err != nil {
		return nil, err
	}

	return stats, nil
}