fix: HA checks resolve service targets over project network; persist checks

- Health checks for local services dialed 127.0.0.1 from inside the backend
  container — the host's published port is unreachable there. Service-bound
  checks now resolve the service's DNS name on its project network (backend
  joins on demand), falling back to the service's stored port. A resolver
  failure surfaces as TARGET_UNRESOLVED instead of a fake loopback probe.
- Checks lived only in memory: a backend restart silently deleted every
  configured health check. ha_health_checks table + write-through on
  add/update/tick and hydrate-on-boot, matching the scaling-policy pattern.
- sqlc schema.sql gained the runtime columns added by later migrations —
  the stale snapshot already caused one silently-dropped-column bug.
- OpenAPI: runtime response now documents ports and health fields; types
  regenerated.
- Buttons no longer wrap ("+ new project" rendered on two lines).

Verified live: service check probes whoami-web:80 via DNS → healthy;
check survives a backend restart and rehydrates.

Generated with [Devin](https://devin.ai)

Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com>

app/backend/internal/api/routes.go

@@ -57,6 +57,33 @@ func SetupRoutes(router *gin.Engine, db *database.DB, redis *database.Redis, cfg
 		}
 	}
 	haManager := ha.NewHighAvailabilityManager(scheduler, metricsCollector)
+	if db != nil && db.DB != nil {
+		haManager.WithPersistence(db.DB)
+		if err := haManager.LoadHealthChecks(context.Background()); err != nil {
+			log.Printf("Failed to restore health checks: %v", err)
+		}
+	}
+	if dockerClient != nil && db != nil {
+		// Service-bound checks probe the service DNS name on the project
+		// network — the backend container joins it on demand. The old
+		// 127.0.0.1 fallback stays for node-bound checks.
+		docker := dockerClient
+		haManager.WithServiceResolver(func(ctx context.Context, serviceID string) (string, int, error) {
+			var name, projectID string
+			var port int
+			err := db.QueryRowContext(ctx,
+				`SELECT name, project_id::text, COALESCE(port, 0) FROM services WHERE id = $1`,
+				serviceID,
+			).Scan(&name, &projectID, &port)
+			if err != nil {
+				return "", 0, err
+			}
+			if err := docker.ConnectSelfToNetwork(ctx, deployment.ProjectNetworkName(projectID)); err != nil {
+				return "", 0, err
+			}
+			return name, port, nil
+		})
+	}
 	haAPIManager := NewHAManager(haManager)
 
 	// Initialize scaling handler

app/backend/internal/deployment/runtime.go

@@ -67,7 +67,9 @@ type RuntimeState struct {
 	Status     string             `json:"status"`           // running, degraded, stopped
 }
 
-func projectNetworkName(projectID string) string {
+// ProjectNetworkName is the deterministic docker network every service in a
+// project joins — containr-proj-<first 12 hex chars of the project uuid>.
+func ProjectNetworkName(projectID string) string {
 	short := strings.ReplaceAll(projectID, "-", "")
 	if len(short) > 12 {
 		short = short[:12]
@@ -82,7 +84,7 @@ func serviceNamePrefix(serviceID string) string {
 // EnsureProjectNetwork creates the per-project bridge network if missing.
 // Containers on it resolve each other by service name (private networking).
 func (de *DeploymentEngine) EnsureProjectNetwork(ctx context.Context, projectID string) (string, error) {
-	name := projectNetworkName(projectID)
+	name := ProjectNetworkName(projectID)
 
 	networks, err := de.dockerClient.ListNetworks(ctx)
 	if err != nil {
@@ -364,7 +366,7 @@ func (de *DeploymentEngine) ProbeServiceHealth(ctx context.Context, state *Runti
 	}
 
 	probeURL := ""
-	if networkName := projectNetworkName(projectID); networkName != "" {
+	if networkName := ProjectNetworkName(projectID); networkName != "" {
 		if err := de.dockerClient.ConnectSelfToNetwork(ctx, networkName); err == nil {
 			probeURL = fmt.Sprintf("http://%s:%d%s", serviceName, port, healthPath)
 		}
@@ -456,7 +458,7 @@ func (de *DeploymentEngine) RemoveProjectContainers(ctx context.Context, project
 
 // RemoveProjectNetwork removes the per-project bridge network if present.
 func (de *DeploymentEngine) RemoveProjectNetwork(ctx context.Context, projectID string) error {
-	return de.dockerClient.RemoveNetwork(ctx, projectNetworkName(projectID))
+	return de.dockerClient.RemoveNetwork(ctx, ProjectNetworkName(projectID))
 }
 
 func replicaIndex(c container.Summary) int {

app/backend/internal/ha/manager.go

@@ -3,6 +3,7 @@ package ha
 import (
 	"bytes"
 	"context"
+	"database/sql"
 	"encoding/json"
 	"fmt"
 	"log"
@@ -75,11 +76,13 @@ type HealthCheckConfig struct {
 
 // HealthChecker performs health checks on services and nodes
 type HealthChecker struct {
-	scheduler     *deployment.Scheduler
-	checks        map[string]*HealthCheck
-	results       map[string]*HealthCheckResult
-	mu            sync.RWMutex
-	checkInterval time.Duration
+	scheduler        *deployment.Scheduler
+	checks           map[string]*HealthCheck
+	results          map[string]*HealthCheckResult
+	mu               sync.RWMutex
+	checkInterval    time.Duration
+	serviceResolver  func(ctx context.Context, serviceID string) (host string, port int, err error)
+	db               *sql.DB
 }
 
 // HealthCheck represents a health check configuration
@@ -596,7 +599,11 @@ func (hc *HealthChecker) performHealthChecks(ctx context.Context) error {
 		result := hc.performHealthCheck(ctx, check)
 		hc.mu.Lock()
 		hc.results[check.ID] = result
+		check.Status = result.Status
+		check.LastCheck = result.Timestamp
 		hc.mu.Unlock()
+		// Persist last-known state so a restart resumes with honest history.
+		hc.persistCheck(check)
 	}
 
 	return nil
@@ -617,12 +624,20 @@ func (hc *HealthChecker) performHealthCheck(ctx context.Context, check *HealthCh
 	checkCtx, cancel := context.WithTimeout(ctx, timeout)
 	defer cancel()
 
-	var err error
+	host, port, err := hc.resolveTarget(ctx, check)
+	if err != nil {
+		result.Status = HealthStatusUnhealthy
+		result.Message = err.Error()
+		result.ErrorCode = "TARGET_UNRESOLVED"
+		result.Latency = time.Since(start)
+		return result
+	}
+
 	switch check.Type {
 	case HealthCheckTypeHTTP:
-		err = hc.performHTTPCheck(checkCtx, check)
+		err = hc.performHTTPCheck(checkCtx, check, host, port)
 	case HealthCheckTypeTCP:
-		err = hc.performTCPCheck(checkCtx, check)
+		err = hc.performTCPCheck(checkCtx, check, host, port)
 	case HealthCheckTypeCommand:
 		err = hc.performCommandCheck(checkCtx, check)
 	default:
@@ -642,13 +657,11 @@ func (hc *HealthChecker) performHealthCheck(ctx context.Context, check *HealthCh
 	return result
 }
 
-func (hc *HealthChecker) performHTTPCheck(ctx context.Context, check *HealthCheck) error {
-	host := hc.resolveHealthCheckHost(check)
+func (hc *HealthChecker) performHTTPCheck(ctx context.Context, check *HealthCheck, host string, port int) error {
 	protocol := strings.ToLower(strings.TrimSpace(check.Config.Protocol))
 	if protocol == "" {
 		protocol = "http"
 	}
-	port := check.Config.Port
 	if port <= 0 {
 		if protocol == "https" {
 			port = 443
@@ -684,9 +697,7 @@ func (hc *HealthChecker) performHTTPCheck(ctx context.Context, check *HealthChec
 	return nil
 }
 
-func (hc *HealthChecker) performTCPCheck(ctx context.Context, check *HealthCheck) error {
-	host := hc.resolveHealthCheckHost(check)
-	port := check.Config.Port
+func (hc *HealthChecker) performTCPCheck(ctx context.Context, check *HealthCheck, host string, port int) error {
 	if port <= 0 {
 		port = 80
 	}
@@ -723,6 +734,27 @@ func (hc *HealthChecker) performCommandCheck(ctx context.Context, check *HealthC
 	return nil
 }
 
+// WithServiceResolver lets the API layer plug in service-aware targeting: a
+// ServiceID-bound check resolves to the service's DNS name on its project
+// network instead of the backend container's loopback, where dialing
+// 127.0.0.1:<host port> can never succeed.
+func (m *HighAvailabilityManager) WithServiceResolver(fn func(ctx context.Context, serviceID string) (host string, port int, err error)) *HighAvailabilityManager {
+	m.healthChecker.serviceResolver = fn
+	return m
+}
+
+// WithPersistence attaches Postgres so configured checks survive restarts.
+// Call LoadHealthChecks once during startup to hydrate state.
+func (m *HighAvailabilityManager) WithPersistence(db *sql.DB) *HighAvailabilityManager {
+	m.healthChecker.db = db
+	return m
+}
+
+// LoadHealthChecks hydrates persisted health checks during startup.
+func (m *HighAvailabilityManager) LoadHealthChecks(ctx context.Context) error {
+	return m.healthChecker.LoadHealthChecks(ctx)
+}
+
 func (hc *HealthChecker) resolveHealthCheckHost(check *HealthCheck) string {
 	if strings.TrimSpace(check.NodeID) != "" && hc.scheduler != nil {
 		nodes := hc.scheduler.GetNodes()
@@ -739,19 +771,109 @@ func (hc *HealthChecker) resolveHealthCheckHost(check *HealthCheck) string {
 	return "127.0.0.1"
 }
 
+// resolveTarget picks the dial target for a check: service-bound checks go to
+// the service DNS name on the project network; node-bound and free-standing
+// checks keep the old host behavior. A service-bound resolver failure is
+// surfaced as an error — falling back to loopback would only produce a
+// misleading "service down" result.
+func (hc *HealthChecker) resolveTarget(ctx context.Context, check *HealthCheck) (string, int, error) {
+	port := check.Config.Port
+	if strings.TrimSpace(check.NodeID) == "" && check.ServiceID != "" && hc.serviceResolver != nil {
+		resolveCtx, cancel := context.WithTimeout(ctx, 10*time.Second)
+		defer cancel()
+		h, p, err := hc.serviceResolver(resolveCtx, check.ServiceID)
+		if err != nil {
+			return "", 0, fmt.Errorf("resolve service target: %w", err)
+		}
+		if h == "" {
+			return "", 0, fmt.Errorf("resolve service target: service %s not found", check.ServiceID)
+		}
+		if port <= 0 {
+			port = p
+		}
+		return h, port, nil
+	}
+	return hc.resolveHealthCheckHost(check), port, nil
+}
+
 // AddHealthCheck adds a new health check
 func (hc *HealthChecker) AddHealthCheck(check *HealthCheck) {
 	hc.mu.Lock()
-	defer hc.mu.Unlock()
 	hc.checks[check.ID] = check
+	hc.mu.Unlock()
+	hc.persistCheck(check)
 }
 
 // RemoveHealthCheck removes a health check
 func (hc *HealthChecker) RemoveHealthCheck(checkID string) {
 	hc.mu.Lock()
-	defer hc.mu.Unlock()
 	delete(hc.checks, checkID)
 	delete(hc.results, checkID)
+	hc.mu.Unlock()
+	if hc.db != nil {
+		_, _ = hc.db.ExecContext(context.Background(), `DELETE FROM ha_health_checks WHERE id = $1`, checkID)
+	}
+}
+
+// persistCheck upserts one check. Best effort — a write failure never blocks
+// the in-memory path, matching the scaling-policy persistence style.
+func (hc *HealthChecker) persistCheck(check *HealthCheck) {
+	if hc.db == nil {
+		return
+	}
+	cfg, _ := json.Marshal(check.Config)
+	var lastCheck interface{}
+	if !check.LastCheck.IsZero() {
+		lastCheck = check.LastCheck
+	}
+	_, _ = hc.db.ExecContext(context.Background(),
+		`INSERT INTO ha_health_checks (id, service_id, node_id, type, config, status, last_check, updated_at)
+		 VALUES ($1, $2, $3, $4, $5, $6, $7, now())
+		 ON CONFLICT (id) DO UPDATE
+		 SET service_id = EXCLUDED.service_id, node_id = EXCLUDED.node_id,
+		     type = EXCLUDED.type, config = EXCLUDED.config,
+		     status = EXCLUDED.status, last_check = EXCLUDED.last_check,
+		     updated_at = now()`,
+		check.ID, check.ServiceID, check.NodeID, string(check.Type), cfg, string(check.Status), lastCheck,
+	)
+}
+
+// LoadHealthChecks hydrates checks persisted across restarts.
+func (hc *HealthChecker) LoadHealthChecks(ctx context.Context) error {
+	if hc.db == nil {
+		return nil
+	}
+	rows, err := hc.db.QueryContext(ctx,
+		`SELECT id, service_id, node_id, type, config, status, last_check FROM ha_health_checks`)
+	if err != nil {
+		return err
+	}
+	defer rows.Close()
+
+	loaded := 0
+	for rows.Next() {
+		var check HealthCheck
+		var cfg []byte
+		var status, checkType string
+		var lastCheck sql.NullTime
+		if err := rows.Scan(&check.ID, &check.ServiceID, &check.NodeID, &checkType, &cfg, &status, &lastCheck); err != nil {
+			continue
+		}
+		check.Type = HealthCheckType(checkType)
+		check.Status = HealthStatus(status)
+		if lastCheck.Valid {
+			check.LastCheck = lastCheck.Time
+		}
+		_ = json.Unmarshal(cfg, &check.Config)
+		hc.mu.Lock()
+		hc.checks[check.ID] = &check
+		hc.mu.Unlock()
+		loaded++
+	}
+	if loaded > 0 {
+		log.Printf("Restored %d health checks", loaded)
+	}
+	return rows.Err()
 }
 
 // GetHealthCheck returns one configured health check.

app/backend/migrations_goose/20261004120000_ha_health_checks.sql

@@ -0,0 +1,18 @@
+-- +goose Up
+-- Persisted HA health checks. Previously checks lived only in memory, so a
+-- backend restart silently deleted every configured check while services kept
+-- running unmonitored.
+CREATE TABLE IF NOT EXISTS ha_health_checks (
+    id          uuid PRIMARY KEY,
+    service_id  uuid NOT NULL REFERENCES services(id) ON DELETE CASCADE,
+    node_id     varchar(255) NOT NULL DEFAULT '',
+    type        varchar(20) NOT NULL,
+    config      jsonb NOT NULL DEFAULT '{}'::jsonb,
+    status      varchar(20) NOT NULL DEFAULT 'unknown',
+    last_check  timestamptz,
+    created_at  timestamptz NOT NULL DEFAULT now(),
+    updated_at  timestamptz NOT NULL DEFAULT now()
+);
+
+-- +goose Down
+DROP TABLE IF EXISTS ha_health_checks;

app/backend/sqlc/schema.sql

@@ -52,7 +52,13 @@ CREATE TABLE services (
     git_branch VARCHAR(100),
     build_path VARCHAR(500),
     cpu VARCHAR(50),
-    memory VARCHAR(50)
+    memory VARCHAR(50),
+    replicas INTEGER NOT NULL DEFAULT 1,
+    port INTEGER NOT NULL DEFAULT 0,
+    domain VARCHAR(255) NOT NULL DEFAULT '',
+    healthcheck_path VARCHAR(255) NOT NULL DEFAULT '',
+    restart_policy VARCHAR(50) NOT NULL DEFAULT 'unless-stopped',
+    published_port INTEGER NOT NULL DEFAULT 0
 );
 
 CREATE TABLE deployments (

app/frontend/src/generated/api-types.ts

@@ -637,7 +637,15 @@ export interface paths {
                                 desired?: number;
                                 /** @enum {string} */
                                 status?: "running" | "degraded" | "stopped";
+                                /** @description Reachable service URLs; rewritten to the request Host for remote clients. Null/empty when nothing is published. */
                                 urls?: string[];
+                                /** @description Host ports actually bound for replica 0+; persisted on the service so restarts reuse the same public port. Null/empty when nothing is published. */
+                                ports?: number[];
+                                /**
+                                 * @description Result of the backend's HTTP probe of the service on its project network. Omitted when the service declares no healthcheck path.
+                                 * @enum {string}
+                                 */
+                                health?: "healthy" | "unhealthy" | "unreachable";
                                 containers?: {
                                     id?: string;
                                     name?: string;

app/frontend/src/index.css

@@ -1523,6 +1523,7 @@ textarea {
   background: var(--accent-primary); color: var(--accent-on);
   border: none; border-radius: 8px; padding: 9px 17px;
   font: 700 12px 'JetBrains Mono', monospace; cursor: pointer;
+  white-space: nowrap; flex-shrink: 0;
   transition: background 0.15s, transform 0.15s;
   box-shadow: 0 2px 12px rgba(180, 227, 74, 0.22);
 }
@@ -1534,6 +1535,7 @@ textarea {
   background: var(--surface-card); color: var(--text-secondary);
   border: 1px solid var(--border-subtle); border-radius: 8px; padding: 8px 16px;
   font: 600 12px 'JetBrains Mono', monospace; cursor: pointer; transition: 0.15s;
+  white-space: nowrap; flex-shrink: 0;
 }
 .v-btn-ghost:hover { color: var(--text-primary); border-color: var(--border-default); }
 

docs/api/openapi.yaml

@@ -458,6 +458,16 @@ paths:
                         type: array
                         items:
                           type: string
+                        description: Reachable service URLs; rewritten to the request Host for remote clients. Null/empty when nothing is published.
+                      ports:
+                        type: array
+                        items:
+                          type: integer
+                        description: Host ports actually bound for replica 0+; persisted on the service so restarts reuse the same public port. Null/empty when nothing is published.
+                      health:
+                        type: string
+                        enum: [healthy, unhealthy, unreachable]
+                        description: Result of the backend's HTTP probe of the service on its project network. Omitted when the service declares no healthcheck path.
                       containers:
                         type: array
                         items: