diff --git a/ROADMAP.md b/ROADMAP.md index c36c514..1e5fb0a 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -51,7 +51,7 @@ The backend exposes **~95 route registrations**; the frontend consumes | Preview environments (CRUD, promote, cleanup) | done (bookkeeping only — no real container deploy; URL is a placeholder) | done — Previews section on service detail | OPERATIONS.md | | Security scans, vulnerabilities, compliance/GDPR reports | done (heuristic scanner — image tags, insecure URLs, build-command smells; no Trivy/Grype binary scanning) | done — `/security` page (scan trigger, findings, metrics, history) | OPERATIONS.md | | Audit logs | done | done — `/settings/audit-logs` filterable page | done | -| Autoscaling policies + manual scale | done | done (beta) — per-service Scaling section; policies in-memory, lost on restart | AUTOSCALING.md | +| Autoscaling policies + manual scale | done | done — per-service Scaling section; policies persisted (scale events still in-memory) | AUTOSCALING.md | | HA / failover policies | done | done (beta) | OPERATIONS.md | | Node agents (register additional VPS/LXC/VM hosts, heartbeats) | done | done — token issue/revoke + install command on Usage page | done | | API gateway (merged APwhy): upstream services, API keys, rate limits, ops/traffic analytics | management API done — native handlers under `/api/v1/gateway/*`, schema-corrected; **traffic proxy path still not mounted** (Phase 2) | **missing** | not documented | @@ -364,8 +364,8 @@ reaches the upstream container and appears in the analytics tables. on service detail Scaling section. Verified live: policy CRUD, state auto-registration, bounds rejection, scheduler dispatch (fails "no ready nodes" without agents — error surfaces in UI). - Scaling history timeline not surfaced. Beta: policies are - in-memory — they do not survive a backend restart. + Scaling history timeline not surfaced. Policies now persisted to + Postgres (survive restart); scale events remain in-memory. - [x] **HA / failover UI** (`/ha/*`): `/ha` page with manager enable/disable, status cards (nodes/health checks/alerts), manual failover with confirmation, per-service failover policy CRUD (project→service @@ -441,10 +441,12 @@ security scans produce real reports, not empty tables. call sites across ~14 handler files remain (agents + tokens are sqlc). Mechanical port to `sqlc/queries/`; the consolidated baseline schema is now the codegen source of truth. -- [ ] **Autoscaling persistence**: policies, service states, and scale - events live in process memory — a backend restart wipes them and - reverts replicas to policy minimums on next registration. Persist - to Postgres before leaving beta. +- [x] **Autoscaling persistence**: `scaling_policies` table + autoscaler + `WithPersistence`/`LoadPolicies` — policies upsert on every set and + hydrate on boot, re-registering service state at `min_replicas`. + Verified: policy survives a backend restart. Scale event history and + placement maps remain in-memory (history resets on restart — + acceptable for beta). - [x] **Demo-data parity for new pages**: `demoDatabases` fixture wired into `/databases?demo=1`; `demoCronJobsByService` fixtures wired into the service Cron tab (worker + api demo services). Scaling, Previews, diff --git a/app/backend/internal/api/databases.go b/app/backend/internal/api/databases.go index 35c728e..1f0ea0b 100644 --- a/app/backend/internal/api/databases.go +++ b/app/backend/internal/api/databases.go @@ -40,21 +40,21 @@ const ( // DatabaseService represents a managed database service type DatabaseService struct { - ID string `json:"id" db:"id"` - Name string `json:"name" db:"name"` - Type string `json:"type" db:"type"` // postgresql, redis, mysql, mariadb, mongodb, clickhouse, dragonfly - Status string `json:"status" db:"status"` // running, stopped, building, error - Version string `json:"version" db:"version"` - Plan string `json:"plan" db:"plan"` // hobby, starter, standard, business - Region string `json:"region" db:"region"` - BackupSchedule string `json:"backup_schedule,omitempty"` - NextBackupAt *time.Time `json:"next_backup_at,omitempty"` - CreatedAt time.Time `json:"created_at" db:"created_at"` - UpdatedAt time.Time `json:"updated_at" db:"updated_at"` - ConnectionURL string `json:"connection_url"` - Metrics DatabaseMetrics `json:"metrics"` - Backups DatabaseBackupConfig `json:"backups"` - Settings DatabaseSettings `json:"settings"` + ID string `json:"id" db:"id"` + Name string `json:"name" db:"name"` + Type string `json:"type" db:"type"` // postgresql, redis, mysql, mariadb, mongodb, clickhouse, dragonfly + Status string `json:"status" db:"status"` // running, stopped, building, error + Version string `json:"version" db:"version"` + Plan string `json:"plan" db:"plan"` // hobby, starter, standard, business + Region string `json:"region" db:"region"` + BackupSchedule string `json:"backup_schedule,omitempty"` + NextBackupAt *time.Time `json:"next_backup_at,omitempty"` + CreatedAt time.Time `json:"created_at" db:"created_at"` + UpdatedAt time.Time `json:"updated_at" db:"updated_at"` + ConnectionURL string `json:"connection_url"` + Metrics DatabaseMetrics `json:"metrics"` + Backups DatabaseBackupConfig `json:"backups"` + Settings DatabaseSettings `json:"settings"` } // DatabaseMetrics represents database performance metrics @@ -1040,7 +1040,7 @@ func (h *DatabaseHandler) resolveManagedRuntimeStatus(ctx context.Context, db Da func mapDatabaseServiceListRow(row sqlcdb.ListDatabaseServicesByUserRow) DatabaseService { return DatabaseService{ - ID: row.ID, + ID: row.ID, Name: row.Name, Type: row.Type, Status: row.Status, diff --git a/app/backend/internal/api/routes.go b/app/backend/internal/api/routes.go index 431fe42..0fdc7f4 100644 --- a/app/backend/internal/api/routes.go +++ b/app/backend/internal/api/routes.go @@ -48,6 +48,12 @@ func SetupRoutes(router *gin.Engine, db *database.DB, redis *database.Redis, cfg } metricsCollector := metrics.NewMetricsCollector(scheduler, metricsStorage) autoScaler := scaling.NewAutoScaler(scheduler, metricsCollector) + if db != nil && db.DB != nil { + autoScaler.WithPersistence(db.DB) + if err := autoScaler.LoadPolicies(context.Background()); err != nil { + log.Printf("Failed to restore scaling policies: %v", err) + } + } haManager := ha.NewHighAvailabilityManager(scheduler, metricsCollector) haAPIManager := NewHAManager(haManager) diff --git a/app/backend/internal/scaling/autoscaler.go b/app/backend/internal/scaling/autoscaler.go index 4ed08cd..2fc06cf 100644 --- a/app/backend/internal/scaling/autoscaler.go +++ b/app/backend/internal/scaling/autoscaler.go @@ -2,6 +2,8 @@ package scaling import ( "context" + "database/sql" + "encoding/json" "fmt" "log" "math" @@ -21,6 +23,7 @@ type AutoScaler struct { servicePlacements map[string]map[string]string // service_id -> instance_id -> node_id replicaSequence map[string]int64 events []ScaleEvent + persist *sql.DB mu sync.RWMutex checkInterval time.Duration cooldownPeriod time.Duration @@ -114,6 +117,119 @@ func NewAutoScaler(scheduler *deployment.Scheduler, metricsCollector *metrics.Me } } +// WithPersistence attaches a Postgres handle so scaling policies survive +// restarts. Call LoadPolicies once during startup to hydrate state. +func (as *AutoScaler) WithPersistence(db *sql.DB) { + as.mu.Lock() + defer as.mu.Unlock() + as.persist = db +} + +// LoadPolicies hydrates policies (and matching service states) from +// scaling_policies. Missing/deleted services are skipped via the FK. +func (as *AutoScaler) LoadPolicies(ctx context.Context) error { + as.mu.RLock() + db := as.persist + as.mu.RUnlock() + if db == nil { + return nil + } + + rows, err := db.QueryContext(ctx, ` + SELECT service_id, min_replicas, max_replicas, target_cpu, target_memory, + scale_up_cooldown, scale_down_cooldown, scale_up_step, scale_down_step, + metrics, thresholds, enabled, cost_optimization + FROM scaling_policies`) + if err != nil { + return fmt.Errorf("load scaling policies: %w", err) + } + defer rows.Close() + + var loaded []*ScalingPolicy + for rows.Next() { + var ( + p ScalingPolicy + metricsRaw, thresholdsRaw []byte + costRaw []byte + ) + if err := rows.Scan( + &p.ServiceID, &p.MinReplicas, &p.MaxReplicas, &p.TargetCPU, &p.TargetMemory, + &p.ScaleUpCooldown, &p.ScaleDownCooldown, &p.ScaleUpStep, &p.ScaleDownStep, + &metricsRaw, &thresholdsRaw, &p.Enabled, &costRaw, + ); err != nil { + return fmt.Errorf("scan scaling policy: %w", err) + } + if len(metricsRaw) > 0 { + _ = json.Unmarshal(metricsRaw, &p.Metrics) + } + if len(thresholdsRaw) > 0 { + _ = json.Unmarshal(thresholdsRaw, &p.Thresholds) + } + if len(costRaw) > 0 { + var cost CostOptimization + if json.Unmarshal(costRaw, &cost) == nil { + p.CostOptimization = &cost + } + } + loaded = append(loaded, &p) + } + if err := rows.Err(); err != nil { + return err + } + + for _, policy := range loaded { + if err := as.SetScalingPolicy(policy); err != nil { + log.Printf("scaling: failed to restore policy for %s: %v", policy.ServiceID, err) + } + } + return nil +} + +// persistPolicy upserts a policy row. Best-effort: the in-memory policy is +// authoritative; a persistence failure is logged, not fatal. +func (as *AutoScaler) persistPolicy(policy *ScalingPolicy) { + as.mu.RLock() + db := as.persist + as.mu.RUnlock() + if db == nil { + return + } + + metricsRaw, _ := json.Marshal(policy.Metrics) + thresholdsRaw, _ := json.Marshal(policy.Thresholds) + costRaw, _ := json.Marshal(policy.CostOptimization) + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + _, err := db.ExecContext(ctx, ` + INSERT INTO scaling_policies ( + service_id, min_replicas, max_replicas, target_cpu, target_memory, + scale_up_cooldown, scale_down_cooldown, scale_up_step, scale_down_step, + metrics, thresholds, enabled, cost_optimization, updated_at + ) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13, now()) + ON CONFLICT (service_id) DO UPDATE SET + min_replicas = EXCLUDED.min_replicas, + max_replicas = EXCLUDED.max_replicas, + target_cpu = EXCLUDED.target_cpu, + target_memory = EXCLUDED.target_memory, + scale_up_cooldown = EXCLUDED.scale_up_cooldown, + scale_down_cooldown = EXCLUDED.scale_down_cooldown, + scale_up_step = EXCLUDED.scale_up_step, + scale_down_step = EXCLUDED.scale_down_step, + metrics = EXCLUDED.metrics, + thresholds = EXCLUDED.thresholds, + enabled = EXCLUDED.enabled, + cost_optimization = EXCLUDED.cost_optimization, + updated_at = now()`, + policy.ServiceID, policy.MinReplicas, policy.MaxReplicas, policy.TargetCPU, policy.TargetMemory, + int64(policy.ScaleUpCooldown), int64(policy.ScaleDownCooldown), policy.ScaleUpStep, policy.ScaleDownStep, + metricsRaw, thresholdsRaw, policy.Enabled, costRaw, + ) + if err != nil { + log.Printf("scaling: failed to persist policy for %s: %v", policy.ServiceID, err) + } +} + // Start begins the auto-scaling process func (as *AutoScaler) Start(ctx context.Context) error { ticker := time.NewTicker(as.checkInterval) @@ -783,6 +899,8 @@ func (as *AutoScaler) SetScalingPolicy(policy *ScalingPolicy) error { as.services[policy.ServiceID].Policy = policy } + go as.persistPolicy(policy) + return nil } diff --git a/app/backend/migrations_goose/20260921000000_scaling_policies.sql b/app/backend/migrations_goose/20260921000000_scaling_policies.sql new file mode 100644 index 0000000..bd67826 --- /dev/null +++ b/app/backend/migrations_goose/20260921000000_scaling_policies.sql @@ -0,0 +1,20 @@ +-- +goose Up +CREATE TABLE IF NOT EXISTS scaling_policies ( + service_id uuid PRIMARY KEY REFERENCES services(id) ON DELETE CASCADE, + min_replicas integer NOT NULL DEFAULT 1, + max_replicas integer NOT NULL DEFAULT 1, + target_cpu double precision NOT NULL DEFAULT 0, + target_memory double precision NOT NULL DEFAULT 0, + scale_up_cooldown bigint NOT NULL DEFAULT 0, + scale_down_cooldown bigint NOT NULL DEFAULT 0, + scale_up_step integer NOT NULL DEFAULT 1, + scale_down_step integer NOT NULL DEFAULT 1, + metrics jsonb, + thresholds jsonb, + enabled boolean NOT NULL DEFAULT true, + cost_optimization jsonb, + updated_at timestamptz NOT NULL DEFAULT now() +); + +-- +goose Down +DROP TABLE IF EXISTS scaling_policies;