feat(nodes): agent-reported container liveness at heartbeat

The agent now lists every containr-managed container's docker state at
each heartbeat; the server reconciles container_instances for rows that
node owns, so exits, OOMs, and crash loops surface in runtime state
between reconciles. Tombstoned rows are never resurrected.

Generated with [Devin](https://devin.ai)

Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
Tomas DvorakandDevin committed 2026-10-09 18:29:48 +02:00
1 parent c3a35081d0
commit 4028639239
6 files changed
+84 -6

No files matched your search

+23
View File
@@ -198,6 +198,15 @@ type AgentHeartbeat struct {
SystemLoad SystemLoad `json:"system_load"`
Uptime int64 `json:"uptime"`
Version string `json:"version"`
// Containers reports managed-container liveness — {name, state} per
// containr.managed container on the node — so remote replica status
// reflects reality between reconciles (exits, OOMs, crash loops).
Containers []AgentContainerState `json:"containers,omitempty"`
}
type AgentContainerState struct {
Name string `json:"name"`
State string `json:"state"`
}
type SystemLoad struct {
@@ -665,6 +674,20 @@ func (h *NodeAgentHandler) SendHeartbeat(c *gin.Context) {
return
}
// Reconcile remote container liveness — exits and crash loops surface
// in inventory between reconciles. Best-effort, never fatal.
for _, rc := range heartbeat.Containers {
if rc.Name == "" || rc.State == "" {
continue
}
stateJSON, _ := json.Marshal(rc.State)
_ = h.q.UpdateContainerStateByName(ctx, sqlcdb.UpdateContainerStateByNameParams{
Name: rc.Name,
NodeAgentID: heartbeat.NodeAgentID,
Column3: stateJSON,
})
}
c.Status(http.StatusOK)
}
@@ -1053,6 +1053,27 @@ func (q *Queries) UpdateContainerPorts(ctx context.Context, arg UpdateContainerP
return err
}
const updateContainerStateByName = `-- name: UpdateContainerStateByName :exec
UPDATE container_instances SET
status = jsonb_set(status, '{state}', $3::jsonb),
updated_at = NOW()
WHERE name = $1 AND node_agent_id = $2
AND status->>'state' IS DISTINCT FROM 'removed'
`
type UpdateContainerStateByNameParams struct {
Name string `json:"name"`
NodeAgentID string `json:"node_agent_id"`
Column3 json.RawMessage `json:"column_3"`
}
// Agent heartbeat reports real docker state; reconcile inventory for rows
// this node owns. Tombstones win — never resurrect a removed replica.
func (q *Queries) UpdateContainerStateByName(ctx context.Context, arg UpdateContainerStateByNameParams) error {
_, err := q.db.ExecContext(ctx, updateContainerStateByName, arg.Name, arg.NodeAgentID, arg.Column3)
return err
}
const updateContainerStatus = `-- name: UpdateContainerStatus :exec
UPDATE container_instances SET status = $2, updated_at = NOW()
WHERE id = $1
@@ -139,6 +139,9 @@ type Querier interface {
// Agent reports the real host bindings after create — persist them with
// the status flip so ingress can route to the assigned port.
UpdateContainerPorts(ctx context.Context, arg UpdateContainerPortsParams) error
// Agent heartbeat reports real docker state; reconcile inventory for rows
// this node owns. Tombstones win — never resurrect a removed replica.
UpdateContainerStateByName(ctx context.Context, arg UpdateContainerStateByNameParams) error
UpdateContainerStatus(ctx context.Context, arg UpdateContainerStatusParams) error
UpdateDatabaseServiceNameAndPlanByIDAndUser(ctx context.Context, arg UpdateDatabaseServiceNameAndPlanByIDAndUserParams) error
UpdateDatabaseServiceNameByIDAndUser(ctx context.Context, arg UpdateDatabaseServiceNameByIDAndUserParams) error