feat(nodes): agent-reported container liveness at heartbeat

The agent now lists every containr-managed container's docker state at
each heartbeat; the server reconciles container_instances for rows that
node owns, so exits, OOMs, and crash loops surface in runtime state
between reconciles. Tombstoned rows are never resurrected.

Generated with [Devin](https://devin.ai)

Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com>

ROADMAP2.md

@@ -302,9 +302,11 @@ and wired refs, one click or one CLI call.
   `TRAEFIK_DYNAMIC_DIR` (shared volume, `watch=true`) pointing the
   loadbalancer at each replica's mesh IP + published port; route files
   are rewritten on every remote reconcile and removed on teardown or
-  local placement. Remote nodes run registry-pulled
-  images only — git builds and remote sleep are documented ceilings.
-  Tag affinity shipped: `node_agents.tags`
+  local placement. Agents report managed-container states at every
+  heartbeat — exits, OOMs, and crash loops land in inventory between
+  reconciles (tombstones never resurrect). Remote nodes run
+  registry-pulled images only — git builds and remote sleep are
+  documented ceilings. Tag affinity shipped: `node_agents.tags`
   (operator-set via `PUT /agents/:id` / `nodes update --tags`) +
   `services.placement_tags` — `auto` and `spread` only consider
   online schedulable agents carrying every required tag, tags are
@@ -313,9 +315,8 @@ and wired refs, one click or one CLI call.
   and local + tags are rejected, empty array clears. Surfaces:
   `--placement-tags` on services create/update, `--tags` on
   `nodes update`, MCP args, OpenAPI, node-detail tag editor +
-  service-detail placement-tags input. Remaining: deployment
-  status reported back from agent results, spread weighting by
-  real capacity.
+  service-detail placement-tags input. Remaining: build-on-node,
+  remote sleep/wake, spread weighting by real capacity.
 - [x] **Agent self-upgrade** — `POST /agents/:id/upgrade` enqueues a
   `self_upgrade` command carrying platform + sha256 + server version;
   the agent downloads its replacement from `GET /agents/download/

app/backend/cmd/agent/main.go

@@ -204,10 +204,31 @@ func sendHeartbeat(ctx context.Context, client *http.Client, cfg agentConfig, ag
 		"system_load":     readSystemLoad(),
 		"uptime":          readSystemUptime(),
 		"version":         firstEnv("CONTAINR_AGENT_VERSION", version),
+		"containers":      managedContainerStates(),
 	}
 	return postJSON(ctx, client, cfg, "/api/agents/heartbeat", payload, nil)
 }
 
+// managedContainerStates lists every containr-managed container's docker
+// state so the server can reflect exits and crash loops between reconciles.
+// One docker call per heartbeat; nil on error (states simply go stale).
+func managedContainerStates() []map[string]string {
+	out, err := runDocker("ps", "-a",
+		"--filter", "label=containr.managed=true",
+		"--format", "{{.Names}}\t{{.State}}")
+	if err != nil {
+		return nil
+	}
+	var states []map[string]string
+	for _, line := range strings.Split(out, "\n") {
+		parts := strings.SplitN(line, "\t", 2)
+		if len(parts) == 2 && parts[0] != "" {
+			states = append(states, map[string]string{"name": parts[0], "state": parts[1]})
+		}
+	}
+	return states
+}
+
 func processCommands(ctx context.Context, client *http.Client, cfg agentConfig, agentID string) error {
 	req, err := http.NewRequestWithContext(ctx, http.MethodGet, fmt.Sprintf("%s/api/agents/%s/commands", cfg.APIURL, agentID), nil)
 	if err != nil {

app/backend/internal/api/agents.go

@@ -198,6 +198,15 @@ type AgentHeartbeat struct {
 	SystemLoad     SystemLoad    `json:"system_load"`
 	Uptime         int64         `json:"uptime"`
 	Version        string        `json:"version"`
+	// Containers reports managed-container liveness — {name, state} per
+	// containr.managed container on the node — so remote replica status
+	// reflects reality between reconciles (exits, OOMs, crash loops).
+	Containers []AgentContainerState `json:"containers,omitempty"`
+}
+
+type AgentContainerState struct {
+	Name  string `json:"name"`
+	State string `json:"state"`
 }
 
 type SystemLoad struct {
@@ -665,6 +674,20 @@ func (h *NodeAgentHandler) SendHeartbeat(c *gin.Context) {
 		return
 	}
 
+	// Reconcile remote container liveness — exits and crash loops surface
+	// in inventory between reconciles. Best-effort, never fatal.
+	for _, rc := range heartbeat.Containers {
+		if rc.Name == "" || rc.State == "" {
+			continue
+		}
+		stateJSON, _ := json.Marshal(rc.State)
+		_ = h.q.UpdateContainerStateByName(ctx, sqlcdb.UpdateContainerStateByNameParams{
+			Name:        rc.Name,
+			NodeAgentID: heartbeat.NodeAgentID,
+			Column3:     stateJSON,
+		})
+	}
+
 	c.Status(http.StatusOK)
 }
 

app/backend/internal/database/sqlcdb/agents.sql.go

@@ -1053,6 +1053,27 @@ func (q *Queries) UpdateContainerPorts(ctx context.Context, arg UpdateContainerP
 	return err
 }
 
+const updateContainerStateByName = `-- name: UpdateContainerStateByName :exec
+UPDATE container_instances SET
+    status = jsonb_set(status, '{state}', $3::jsonb),
+    updated_at = NOW()
+WHERE name = $1 AND node_agent_id = $2
+  AND status->>'state' IS DISTINCT FROM 'removed'
+`
+
+type UpdateContainerStateByNameParams struct {
+	Name        string          `json:"name"`
+	NodeAgentID string          `json:"node_agent_id"`
+	Column3     json.RawMessage `json:"column_3"`
+}
+
+// Agent heartbeat reports real docker state; reconcile inventory for rows
+// this node owns. Tombstones win — never resurrect a removed replica.
+func (q *Queries) UpdateContainerStateByName(ctx context.Context, arg UpdateContainerStateByNameParams) error {
+	_, err := q.db.ExecContext(ctx, updateContainerStateByName, arg.Name, arg.NodeAgentID, arg.Column3)
+	return err
+}
+
 const updateContainerStatus = `-- name: UpdateContainerStatus :exec
 UPDATE container_instances SET status = $2, updated_at = NOW()
 WHERE id = $1

app/backend/internal/database/sqlcdb/querier.go

@@ -139,6 +139,9 @@ type Querier interface {
 	// Agent reports the real host bindings after create — persist them with
 	// the status flip so ingress can route to the assigned port.
 	UpdateContainerPorts(ctx context.Context, arg UpdateContainerPortsParams) error
+	// Agent heartbeat reports real docker state; reconcile inventory for rows
+	// this node owns. Tombstones win — never resurrect a removed replica.
+	UpdateContainerStateByName(ctx context.Context, arg UpdateContainerStateByNameParams) error
 	UpdateContainerStatus(ctx context.Context, arg UpdateContainerStatusParams) error
 	UpdateDatabaseServiceNameAndPlanByIDAndUser(ctx context.Context, arg UpdateDatabaseServiceNameAndPlanByIDAndUserParams) error
 	UpdateDatabaseServiceNameByIDAndUser(ctx context.Context, arg UpdateDatabaseServiceNameByIDAndUserParams) error

app/backend/sqlc/queries/agents.sql

@@ -203,6 +203,15 @@ ON CONFLICT (id) DO UPDATE SET
 -- name: DeleteServiceContainersOnAgent :exec
 DELETE FROM container_instances WHERE node_agent_id = $1 AND service_id = $2;
 
+-- name: UpdateContainerStateByName :exec
+-- Agent heartbeat reports real docker state; reconcile inventory for rows
+-- this node owns. Tombstones win — never resurrect a removed replica.
+UPDATE container_instances SET
+    status = jsonb_set(status, '{state}', $3::jsonb),
+    updated_at = NOW()
+WHERE name = $1 AND node_agent_id = $2
+  AND status->>'state' IS DISTINCT FROM 'removed';
+
 -- name: ListServiceAgents :many
 -- Distinct agents holding inventory rows for a service.
 SELECT DISTINCT node_agent_id FROM container_instances WHERE service_id = $1;