From 4028639239495ba545d33d8f4aa3196d9390cfdf Mon Sep 17 00:00:00 2001 From: Tomas Dvorak Date: Fri, 9 Oct 2026 18:29:48 +0200 Subject: [PATCH] feat(nodes): agent-reported container liveness at heartbeat The agent now lists every containr-managed container's docker state at each heartbeat; the server reconciles container_instances for rows that node owns, so exits, OOMs, and crash loops surface in runtime state between reconciles. Tombstoned rows are never resurrected. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- ROADMAP2.md | 13 ++++++----- app/backend/cmd/agent/main.go | 21 +++++++++++++++++ app/backend/internal/api/agents.go | 23 +++++++++++++++++++ .../internal/database/sqlcdb/agents.sql.go | 21 +++++++++++++++++ .../internal/database/sqlcdb/querier.go | 3 +++ app/backend/sqlc/queries/agents.sql | 9 ++++++++ 6 files changed, 84 insertions(+), 6 deletions(-) diff --git a/ROADMAP2.md b/ROADMAP2.md index 2325ea0..9cc17ce 100644 --- a/ROADMAP2.md +++ b/ROADMAP2.md @@ -302,9 +302,11 @@ and wired refs, one click or one CLI call. `TRAEFIK_DYNAMIC_DIR` (shared volume, `watch=true`) pointing the loadbalancer at each replica's mesh IP + published port; route files are rewritten on every remote reconcile and removed on teardown or - local placement. Remote nodes run registry-pulled - images only — git builds and remote sleep are documented ceilings. - Tag affinity shipped: `node_agents.tags` + local placement. Agents report managed-container states at every + heartbeat — exits, OOMs, and crash loops land in inventory between + reconciles (tombstones never resurrect). Remote nodes run + registry-pulled images only — git builds and remote sleep are + documented ceilings. Tag affinity shipped: `node_agents.tags` (operator-set via `PUT /agents/:id` / `nodes update --tags`) + `services.placement_tags` — `auto` and `spread` only consider online schedulable agents carrying every required tag, tags are @@ -313,9 +315,8 @@ and wired refs, one click or one CLI call. and local + tags are rejected, empty array clears. Surfaces: `--placement-tags` on services create/update, `--tags` on `nodes update`, MCP args, OpenAPI, node-detail tag editor + - service-detail placement-tags input. Remaining: deployment - status reported back from agent results, spread weighting by - real capacity. + service-detail placement-tags input. Remaining: build-on-node, + remote sleep/wake, spread weighting by real capacity. - [x] **Agent self-upgrade** — `POST /agents/:id/upgrade` enqueues a `self_upgrade` command carrying platform + sha256 + server version; the agent downloads its replacement from `GET /agents/download/ diff --git a/app/backend/cmd/agent/main.go b/app/backend/cmd/agent/main.go index b2400d9..a2bafa5 100644 --- a/app/backend/cmd/agent/main.go +++ b/app/backend/cmd/agent/main.go @@ -204,10 +204,31 @@ func sendHeartbeat(ctx context.Context, client *http.Client, cfg agentConfig, ag "system_load": readSystemLoad(), "uptime": readSystemUptime(), "version": firstEnv("CONTAINR_AGENT_VERSION", version), + "containers": managedContainerStates(), } return postJSON(ctx, client, cfg, "/api/agents/heartbeat", payload, nil) } +// managedContainerStates lists every containr-managed container's docker +// state so the server can reflect exits and crash loops between reconciles. +// One docker call per heartbeat; nil on error (states simply go stale). +func managedContainerStates() []map[string]string { + out, err := runDocker("ps", "-a", + "--filter", "label=containr.managed=true", + "--format", "{{.Names}}\t{{.State}}") + if err != nil { + return nil + } + var states []map[string]string + for _, line := range strings.Split(out, "\n") { + parts := strings.SplitN(line, "\t", 2) + if len(parts) == 2 && parts[0] != "" { + states = append(states, map[string]string{"name": parts[0], "state": parts[1]}) + } + } + return states +} + func processCommands(ctx context.Context, client *http.Client, cfg agentConfig, agentID string) error { req, err := http.NewRequestWithContext(ctx, http.MethodGet, fmt.Sprintf("%s/api/agents/%s/commands", cfg.APIURL, agentID), nil) if err != nil { diff --git a/app/backend/internal/api/agents.go b/app/backend/internal/api/agents.go index b7bc75b..58ccb12 100644 --- a/app/backend/internal/api/agents.go +++ b/app/backend/internal/api/agents.go @@ -198,6 +198,15 @@ type AgentHeartbeat struct { SystemLoad SystemLoad `json:"system_load"` Uptime int64 `json:"uptime"` Version string `json:"version"` + // Containers reports managed-container liveness — {name, state} per + // containr.managed container on the node — so remote replica status + // reflects reality between reconciles (exits, OOMs, crash loops). + Containers []AgentContainerState `json:"containers,omitempty"` +} + +type AgentContainerState struct { + Name string `json:"name"` + State string `json:"state"` } type SystemLoad struct { @@ -665,6 +674,20 @@ func (h *NodeAgentHandler) SendHeartbeat(c *gin.Context) { return } + // Reconcile remote container liveness — exits and crash loops surface + // in inventory between reconciles. Best-effort, never fatal. + for _, rc := range heartbeat.Containers { + if rc.Name == "" || rc.State == "" { + continue + } + stateJSON, _ := json.Marshal(rc.State) + _ = h.q.UpdateContainerStateByName(ctx, sqlcdb.UpdateContainerStateByNameParams{ + Name: rc.Name, + NodeAgentID: heartbeat.NodeAgentID, + Column3: stateJSON, + }) + } + c.Status(http.StatusOK) } diff --git a/app/backend/internal/database/sqlcdb/agents.sql.go b/app/backend/internal/database/sqlcdb/agents.sql.go index d863e3f..cee6af6 100644 --- a/app/backend/internal/database/sqlcdb/agents.sql.go +++ b/app/backend/internal/database/sqlcdb/agents.sql.go @@ -1053,6 +1053,27 @@ func (q *Queries) UpdateContainerPorts(ctx context.Context, arg UpdateContainerP return err } +const updateContainerStateByName = `-- name: UpdateContainerStateByName :exec +UPDATE container_instances SET + status = jsonb_set(status, '{state}', $3::jsonb), + updated_at = NOW() +WHERE name = $1 AND node_agent_id = $2 + AND status->>'state' IS DISTINCT FROM 'removed' +` + +type UpdateContainerStateByNameParams struct { + Name string `json:"name"` + NodeAgentID string `json:"node_agent_id"` + Column3 json.RawMessage `json:"column_3"` +} + +// Agent heartbeat reports real docker state; reconcile inventory for rows +// this node owns. Tombstones win — never resurrect a removed replica. +func (q *Queries) UpdateContainerStateByName(ctx context.Context, arg UpdateContainerStateByNameParams) error { + _, err := q.db.ExecContext(ctx, updateContainerStateByName, arg.Name, arg.NodeAgentID, arg.Column3) + return err +} + const updateContainerStatus = `-- name: UpdateContainerStatus :exec UPDATE container_instances SET status = $2, updated_at = NOW() WHERE id = $1 diff --git a/app/backend/internal/database/sqlcdb/querier.go b/app/backend/internal/database/sqlcdb/querier.go index 6b297a6..640631b 100644 --- a/app/backend/internal/database/sqlcdb/querier.go +++ b/app/backend/internal/database/sqlcdb/querier.go @@ -139,6 +139,9 @@ type Querier interface { // Agent reports the real host bindings after create — persist them with // the status flip so ingress can route to the assigned port. UpdateContainerPorts(ctx context.Context, arg UpdateContainerPortsParams) error + // Agent heartbeat reports real docker state; reconcile inventory for rows + // this node owns. Tombstones win — never resurrect a removed replica. + UpdateContainerStateByName(ctx context.Context, arg UpdateContainerStateByNameParams) error UpdateContainerStatus(ctx context.Context, arg UpdateContainerStatusParams) error UpdateDatabaseServiceNameAndPlanByIDAndUser(ctx context.Context, arg UpdateDatabaseServiceNameAndPlanByIDAndUserParams) error UpdateDatabaseServiceNameByIDAndUser(ctx context.Context, arg UpdateDatabaseServiceNameByIDAndUserParams) error diff --git a/app/backend/sqlc/queries/agents.sql b/app/backend/sqlc/queries/agents.sql index f7a32ec..2bbc31e 100644 --- a/app/backend/sqlc/queries/agents.sql +++ b/app/backend/sqlc/queries/agents.sql @@ -203,6 +203,15 @@ ON CONFLICT (id) DO UPDATE SET -- name: DeleteServiceContainersOnAgent :exec DELETE FROM container_instances WHERE node_agent_id = $1 AND service_id = $2; +-- name: UpdateContainerStateByName :exec +-- Agent heartbeat reports real docker state; reconcile inventory for rows +-- this node owns. Tombstones win — never resurrect a removed replica. +UPDATE container_instances SET + status = jsonb_set(status, '{state}', $3::jsonb), + updated_at = NOW() +WHERE name = $1 AND node_agent_id = $2 + AND status->>'state' IS DISTINCT FROM 'removed'; + -- name: ListServiceAgents :many -- Distinct agents holding inventory rows for a service. SELECT DISTINCT node_agent_id FROM container_instances WHERE service_id = $1;