From 15b5b0e6e799b47c6f455eae3dbed30886b8a792 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Dvo=C5=99=C3=A1k?= <150935816+Dvorinka@users.noreply.github.com> Date: Sun, 4 Oct 2026 09:37:47 +0200 Subject: [PATCH] fix: HA checks resolve service targets over project network; persist checks (#13) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Health checks for local services dialed 127.0.0.1 from inside the backend container — the host's published port is unreachable there. Service-bound checks now resolve the service's DNS name on its project network (backend joins on demand), falling back to the service's stored port. A resolver failure surfaces as TARGET_UNRESOLVED instead of a fake loopback probe. - Checks lived only in memory: a backend restart silently deleted every configured health check. ha_health_checks table + write-through on add/update/tick and hydrate-on-boot, matching the scaling-policy pattern. - sqlc schema.sql gained the runtime columns added by later migrations — the stale snapshot already caused one silently-dropped-column bug. - OpenAPI: runtime response now documents ports and health fields; types regenerated. - Buttons no longer wrap ("+ new project" rendered on two lines). Verified live: service check probes whoami-web:80 via DNS → healthy; check survives a backend restart and rehydrates. Generated with [Devin](https://devin.ai) Co-authored-by: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- app/backend/internal/api/routes.go | 27 +++ app/backend/internal/deployment/runtime.go | 10 +- app/backend/internal/ha/manager.go | 154 ++++++++++++++++-- .../20261004120000_ha_health_checks.sql | 18 ++ app/backend/sqlc/schema.sql | 8 +- app/frontend/src/generated/api-types.ts | 8 + app/frontend/src/index.css | 2 + docs/api/openapi.yaml | 10 ++ 8 files changed, 216 insertions(+), 21 deletions(-) create mode 100644 app/backend/migrations_goose/20261004120000_ha_health_checks.sql diff --git a/app/backend/internal/api/routes.go b/app/backend/internal/api/routes.go index 13e3be4..2114bd0 100644 --- a/app/backend/internal/api/routes.go +++ b/app/backend/internal/api/routes.go @@ -57,6 +57,33 @@ func SetupRoutes(router *gin.Engine, db *database.DB, redis *database.Redis, cfg } } haManager := ha.NewHighAvailabilityManager(scheduler, metricsCollector) + if db != nil && db.DB != nil { + haManager.WithPersistence(db.DB) + if err := haManager.LoadHealthChecks(context.Background()); err != nil { + log.Printf("Failed to restore health checks: %v", err) + } + } + if dockerClient != nil && db != nil { + // Service-bound checks probe the service DNS name on the project + // network — the backend container joins it on demand. The old + // 127.0.0.1 fallback stays for node-bound checks. + docker := dockerClient + haManager.WithServiceResolver(func(ctx context.Context, serviceID string) (string, int, error) { + var name, projectID string + var port int + err := db.QueryRowContext(ctx, + `SELECT name, project_id::text, COALESCE(port, 0) FROM services WHERE id = $1`, + serviceID, + ).Scan(&name, &projectID, &port) + if err != nil { + return "", 0, err + } + if err := docker.ConnectSelfToNetwork(ctx, deployment.ProjectNetworkName(projectID)); err != nil { + return "", 0, err + } + return name, port, nil + }) + } haAPIManager := NewHAManager(haManager) // Initialize scaling handler diff --git a/app/backend/internal/deployment/runtime.go b/app/backend/internal/deployment/runtime.go index 19d7833..d73c87b 100644 --- a/app/backend/internal/deployment/runtime.go +++ b/app/backend/internal/deployment/runtime.go @@ -67,7 +67,9 @@ type RuntimeState struct { Status string `json:"status"` // running, degraded, stopped } -func projectNetworkName(projectID string) string { +// ProjectNetworkName is the deterministic docker network every service in a +// project joins — containr-proj-. +func ProjectNetworkName(projectID string) string { short := strings.ReplaceAll(projectID, "-", "") if len(short) > 12 { short = short[:12] @@ -82,7 +84,7 @@ func serviceNamePrefix(serviceID string) string { // EnsureProjectNetwork creates the per-project bridge network if missing. // Containers on it resolve each other by service name (private networking). func (de *DeploymentEngine) EnsureProjectNetwork(ctx context.Context, projectID string) (string, error) { - name := projectNetworkName(projectID) + name := ProjectNetworkName(projectID) networks, err := de.dockerClient.ListNetworks(ctx) if err != nil { @@ -364,7 +366,7 @@ func (de *DeploymentEngine) ProbeServiceHealth(ctx context.Context, state *Runti } probeURL := "" - if networkName := projectNetworkName(projectID); networkName != "" { + if networkName := ProjectNetworkName(projectID); networkName != "" { if err := de.dockerClient.ConnectSelfToNetwork(ctx, networkName); err == nil { probeURL = fmt.Sprintf("http://%s:%d%s", serviceName, port, healthPath) } @@ -456,7 +458,7 @@ func (de *DeploymentEngine) RemoveProjectContainers(ctx context.Context, project // RemoveProjectNetwork removes the per-project bridge network if present. func (de *DeploymentEngine) RemoveProjectNetwork(ctx context.Context, projectID string) error { - return de.dockerClient.RemoveNetwork(ctx, projectNetworkName(projectID)) + return de.dockerClient.RemoveNetwork(ctx, ProjectNetworkName(projectID)) } func replicaIndex(c container.Summary) int { diff --git a/app/backend/internal/ha/manager.go b/app/backend/internal/ha/manager.go index ac3b13d..f45b093 100644 --- a/app/backend/internal/ha/manager.go +++ b/app/backend/internal/ha/manager.go @@ -3,6 +3,7 @@ package ha import ( "bytes" "context" + "database/sql" "encoding/json" "fmt" "log" @@ -75,11 +76,13 @@ type HealthCheckConfig struct { // HealthChecker performs health checks on services and nodes type HealthChecker struct { - scheduler *deployment.Scheduler - checks map[string]*HealthCheck - results map[string]*HealthCheckResult - mu sync.RWMutex - checkInterval time.Duration + scheduler *deployment.Scheduler + checks map[string]*HealthCheck + results map[string]*HealthCheckResult + mu sync.RWMutex + checkInterval time.Duration + serviceResolver func(ctx context.Context, serviceID string) (host string, port int, err error) + db *sql.DB } // HealthCheck represents a health check configuration @@ -596,7 +599,11 @@ func (hc *HealthChecker) performHealthChecks(ctx context.Context) error { result := hc.performHealthCheck(ctx, check) hc.mu.Lock() hc.results[check.ID] = result + check.Status = result.Status + check.LastCheck = result.Timestamp hc.mu.Unlock() + // Persist last-known state so a restart resumes with honest history. + hc.persistCheck(check) } return nil @@ -617,12 +624,20 @@ func (hc *HealthChecker) performHealthCheck(ctx context.Context, check *HealthCh checkCtx, cancel := context.WithTimeout(ctx, timeout) defer cancel() - var err error + host, port, err := hc.resolveTarget(ctx, check) + if err != nil { + result.Status = HealthStatusUnhealthy + result.Message = err.Error() + result.ErrorCode = "TARGET_UNRESOLVED" + result.Latency = time.Since(start) + return result + } + switch check.Type { case HealthCheckTypeHTTP: - err = hc.performHTTPCheck(checkCtx, check) + err = hc.performHTTPCheck(checkCtx, check, host, port) case HealthCheckTypeTCP: - err = hc.performTCPCheck(checkCtx, check) + err = hc.performTCPCheck(checkCtx, check, host, port) case HealthCheckTypeCommand: err = hc.performCommandCheck(checkCtx, check) default: @@ -642,13 +657,11 @@ func (hc *HealthChecker) performHealthCheck(ctx context.Context, check *HealthCh return result } -func (hc *HealthChecker) performHTTPCheck(ctx context.Context, check *HealthCheck) error { - host := hc.resolveHealthCheckHost(check) +func (hc *HealthChecker) performHTTPCheck(ctx context.Context, check *HealthCheck, host string, port int) error { protocol := strings.ToLower(strings.TrimSpace(check.Config.Protocol)) if protocol == "" { protocol = "http" } - port := check.Config.Port if port <= 0 { if protocol == "https" { port = 443 @@ -684,9 +697,7 @@ func (hc *HealthChecker) performHTTPCheck(ctx context.Context, check *HealthChec return nil } -func (hc *HealthChecker) performTCPCheck(ctx context.Context, check *HealthCheck) error { - host := hc.resolveHealthCheckHost(check) - port := check.Config.Port +func (hc *HealthChecker) performTCPCheck(ctx context.Context, check *HealthCheck, host string, port int) error { if port <= 0 { port = 80 } @@ -723,6 +734,27 @@ func (hc *HealthChecker) performCommandCheck(ctx context.Context, check *HealthC return nil } +// WithServiceResolver lets the API layer plug in service-aware targeting: a +// ServiceID-bound check resolves to the service's DNS name on its project +// network instead of the backend container's loopback, where dialing +// 127.0.0.1: can never succeed. +func (m *HighAvailabilityManager) WithServiceResolver(fn func(ctx context.Context, serviceID string) (host string, port int, err error)) *HighAvailabilityManager { + m.healthChecker.serviceResolver = fn + return m +} + +// WithPersistence attaches Postgres so configured checks survive restarts. +// Call LoadHealthChecks once during startup to hydrate state. +func (m *HighAvailabilityManager) WithPersistence(db *sql.DB) *HighAvailabilityManager { + m.healthChecker.db = db + return m +} + +// LoadHealthChecks hydrates persisted health checks during startup. +func (m *HighAvailabilityManager) LoadHealthChecks(ctx context.Context) error { + return m.healthChecker.LoadHealthChecks(ctx) +} + func (hc *HealthChecker) resolveHealthCheckHost(check *HealthCheck) string { if strings.TrimSpace(check.NodeID) != "" && hc.scheduler != nil { nodes := hc.scheduler.GetNodes() @@ -739,19 +771,109 @@ func (hc *HealthChecker) resolveHealthCheckHost(check *HealthCheck) string { return "127.0.0.1" } +// resolveTarget picks the dial target for a check: service-bound checks go to +// the service DNS name on the project network; node-bound and free-standing +// checks keep the old host behavior. A service-bound resolver failure is +// surfaced as an error — falling back to loopback would only produce a +// misleading "service down" result. +func (hc *HealthChecker) resolveTarget(ctx context.Context, check *HealthCheck) (string, int, error) { + port := check.Config.Port + if strings.TrimSpace(check.NodeID) == "" && check.ServiceID != "" && hc.serviceResolver != nil { + resolveCtx, cancel := context.WithTimeout(ctx, 10*time.Second) + defer cancel() + h, p, err := hc.serviceResolver(resolveCtx, check.ServiceID) + if err != nil { + return "", 0, fmt.Errorf("resolve service target: %w", err) + } + if h == "" { + return "", 0, fmt.Errorf("resolve service target: service %s not found", check.ServiceID) + } + if port <= 0 { + port = p + } + return h, port, nil + } + return hc.resolveHealthCheckHost(check), port, nil +} + // AddHealthCheck adds a new health check func (hc *HealthChecker) AddHealthCheck(check *HealthCheck) { hc.mu.Lock() - defer hc.mu.Unlock() hc.checks[check.ID] = check + hc.mu.Unlock() + hc.persistCheck(check) } // RemoveHealthCheck removes a health check func (hc *HealthChecker) RemoveHealthCheck(checkID string) { hc.mu.Lock() - defer hc.mu.Unlock() delete(hc.checks, checkID) delete(hc.results, checkID) + hc.mu.Unlock() + if hc.db != nil { + _, _ = hc.db.ExecContext(context.Background(), `DELETE FROM ha_health_checks WHERE id = $1`, checkID) + } +} + +// persistCheck upserts one check. Best effort — a write failure never blocks +// the in-memory path, matching the scaling-policy persistence style. +func (hc *HealthChecker) persistCheck(check *HealthCheck) { + if hc.db == nil { + return + } + cfg, _ := json.Marshal(check.Config) + var lastCheck interface{} + if !check.LastCheck.IsZero() { + lastCheck = check.LastCheck + } + _, _ = hc.db.ExecContext(context.Background(), + `INSERT INTO ha_health_checks (id, service_id, node_id, type, config, status, last_check, updated_at) + VALUES ($1, $2, $3, $4, $5, $6, $7, now()) + ON CONFLICT (id) DO UPDATE + SET service_id = EXCLUDED.service_id, node_id = EXCLUDED.node_id, + type = EXCLUDED.type, config = EXCLUDED.config, + status = EXCLUDED.status, last_check = EXCLUDED.last_check, + updated_at = now()`, + check.ID, check.ServiceID, check.NodeID, string(check.Type), cfg, string(check.Status), lastCheck, + ) +} + +// LoadHealthChecks hydrates checks persisted across restarts. +func (hc *HealthChecker) LoadHealthChecks(ctx context.Context) error { + if hc.db == nil { + return nil + } + rows, err := hc.db.QueryContext(ctx, + `SELECT id, service_id, node_id, type, config, status, last_check FROM ha_health_checks`) + if err != nil { + return err + } + defer rows.Close() + + loaded := 0 + for rows.Next() { + var check HealthCheck + var cfg []byte + var status, checkType string + var lastCheck sql.NullTime + if err := rows.Scan(&check.ID, &check.ServiceID, &check.NodeID, &checkType, &cfg, &status, &lastCheck); err != nil { + continue + } + check.Type = HealthCheckType(checkType) + check.Status = HealthStatus(status) + if lastCheck.Valid { + check.LastCheck = lastCheck.Time + } + _ = json.Unmarshal(cfg, &check.Config) + hc.mu.Lock() + hc.checks[check.ID] = &check + hc.mu.Unlock() + loaded++ + } + if loaded > 0 { + log.Printf("Restored %d health checks", loaded) + } + return rows.Err() } // GetHealthCheck returns one configured health check. diff --git a/app/backend/migrations_goose/20261004120000_ha_health_checks.sql b/app/backend/migrations_goose/20261004120000_ha_health_checks.sql new file mode 100644 index 0000000..1e3e897 --- /dev/null +++ b/app/backend/migrations_goose/20261004120000_ha_health_checks.sql @@ -0,0 +1,18 @@ +-- +goose Up +-- Persisted HA health checks. Previously checks lived only in memory, so a +-- backend restart silently deleted every configured check while services kept +-- running unmonitored. +CREATE TABLE IF NOT EXISTS ha_health_checks ( + id uuid PRIMARY KEY, + service_id uuid NOT NULL REFERENCES services(id) ON DELETE CASCADE, + node_id varchar(255) NOT NULL DEFAULT '', + type varchar(20) NOT NULL, + config jsonb NOT NULL DEFAULT '{}'::jsonb, + status varchar(20) NOT NULL DEFAULT 'unknown', + last_check timestamptz, + created_at timestamptz NOT NULL DEFAULT now(), + updated_at timestamptz NOT NULL DEFAULT now() +); + +-- +goose Down +DROP TABLE IF EXISTS ha_health_checks; diff --git a/app/backend/sqlc/schema.sql b/app/backend/sqlc/schema.sql index 538b658..b85fe2a 100644 --- a/app/backend/sqlc/schema.sql +++ b/app/backend/sqlc/schema.sql @@ -52,7 +52,13 @@ CREATE TABLE services ( git_branch VARCHAR(100), build_path VARCHAR(500), cpu VARCHAR(50), - memory VARCHAR(50) + memory VARCHAR(50), + replicas INTEGER NOT NULL DEFAULT 1, + port INTEGER NOT NULL DEFAULT 0, + domain VARCHAR(255) NOT NULL DEFAULT '', + healthcheck_path VARCHAR(255) NOT NULL DEFAULT '', + restart_policy VARCHAR(50) NOT NULL DEFAULT 'unless-stopped', + published_port INTEGER NOT NULL DEFAULT 0 ); CREATE TABLE deployments ( diff --git a/app/frontend/src/generated/api-types.ts b/app/frontend/src/generated/api-types.ts index 53d0133..ec8e49b 100644 --- a/app/frontend/src/generated/api-types.ts +++ b/app/frontend/src/generated/api-types.ts @@ -637,7 +637,15 @@ export interface paths { desired?: number; /** @enum {string} */ status?: "running" | "degraded" | "stopped"; + /** @description Reachable service URLs; rewritten to the request Host for remote clients. Null/empty when nothing is published. */ urls?: string[]; + /** @description Host ports actually bound for replica 0+; persisted on the service so restarts reuse the same public port. Null/empty when nothing is published. */ + ports?: number[]; + /** + * @description Result of the backend's HTTP probe of the service on its project network. Omitted when the service declares no healthcheck path. + * @enum {string} + */ + health?: "healthy" | "unhealthy" | "unreachable"; containers?: { id?: string; name?: string; diff --git a/app/frontend/src/index.css b/app/frontend/src/index.css index e0db988..68c365e 100644 --- a/app/frontend/src/index.css +++ b/app/frontend/src/index.css @@ -1523,6 +1523,7 @@ textarea { background: var(--accent-primary); color: var(--accent-on); border: none; border-radius: 8px; padding: 9px 17px; font: 700 12px 'JetBrains Mono', monospace; cursor: pointer; + white-space: nowrap; flex-shrink: 0; transition: background 0.15s, transform 0.15s; box-shadow: 0 2px 12px rgba(180, 227, 74, 0.22); } @@ -1534,6 +1535,7 @@ textarea { background: var(--surface-card); color: var(--text-secondary); border: 1px solid var(--border-subtle); border-radius: 8px; padding: 8px 16px; font: 600 12px 'JetBrains Mono', monospace; cursor: pointer; transition: 0.15s; + white-space: nowrap; flex-shrink: 0; } .v-btn-ghost:hover { color: var(--text-primary); border-color: var(--border-default); } diff --git a/docs/api/openapi.yaml b/docs/api/openapi.yaml index e47cd1d..6a23059 100644 --- a/docs/api/openapi.yaml +++ b/docs/api/openapi.yaml @@ -458,6 +458,16 @@ paths: type: array items: type: string + description: Reachable service URLs; rewritten to the request Host for remote clients. Null/empty when nothing is published. + ports: + type: array + items: + type: integer + description: Host ports actually bound for replica 0+; persisted on the service so restarts reuse the same public port. Null/empty when nothing is published. + health: + type: string + enum: [healthy, unhealthy, unreachable] + description: Result of the backend's HTTP probe of the service on its project network. Omitted when the service declares no healthcheck path. containers: type: array items: