feat(nodes): remote sleep/wake — idle detection + wake-page ingress

Agents report per-container net IO at heartbeat; the sleeper measures
idleness from those counters for remote services, removes replicas via
the agent queue, and repoints the domain route at the backend's new
/internal/wake-page endpoint (replacePath + priority-100 route in the
file provider). First hit serves a reloading page and retriggers the
wake reconcile, which recreates replicas and restores real upstreams.

Manual POST /services/:id/sleep also routes remote; local sleep/wake
flow is unchanged.

Generated with [Devin](https://devin.ai)

Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
Tomas DvorakandDevin committed 2026-10-09 18:43:33 +02:00
1 parent 4028639239
commit 59dfd35583
12 files changed
+261 -34

No files matched your search

+8 -3
View File
@@ -304,9 +304,14 @@ and wired refs, one click or one CLI call.
are rewritten on every remote reconcile and removed on teardown or are rewritten on every remote reconcile and removed on teardown or
local placement. Agents report managed-container states at every local placement. Agents report managed-container states at every
heartbeat — exits, OOMs, and crash loops land in inventory between heartbeat — exits, OOMs, and crash loops land in inventory between
reconciles (tombstones never resurrect). Remote nodes run reconciles (tombstones never resurrect). Remote sleep/wake shipped:
registry-pulled images only — git builds and remote sleep are heartbeats carry per-container net counters, idle remote services
documented ceilings. Tag affinity shipped: `node_agents.tags` lose replicas via agent remove + tombstones, and the route file
repoints at `/internal/wake-page/:id` on the backend — first hit
retriggers the wake reconcile, which recreates replicas and restores
real upstreams. Remote nodes run
registry-pulled images only — git builds are the remaining
documented ceiling. Tag affinity shipped: `node_agents.tags`
(operator-set via `PUT /agents/:id` / `nodes update --tags`) + (operator-set via `PUT /agents/:id` / `nodes update --tags`) +
`services.placement_tags` — `auto` and `spread` only consider `services.placement_tags` — `auto` and `spread` only consider
online schedulable agents carrying every required tag, tags are online schedulable agents carrying every required tag, tags are
+67 -6
View File
@@ -210,25 +210,86 @@ func sendHeartbeat(ctx context.Context, client *http.Client, cfg agentConfig, ag
} }
// managedContainerStates lists every containr-managed container's docker // managedContainerStates lists every containr-managed container's docker
// state so the server can reflect exits and crash loops between reconciles. // state plus cumulative network counters — state keeps runtime honest,
// One docker call per heartbeat; nil on error (states simply go stale). // net bytes let the sleeper detect idle remote services.
func managedContainerStates() []map[string]string { func managedContainerStates() []map[string]interface{} {
out, err := runDocker("ps", "-a", out, err := runDocker("ps", "-a",
"--filter", "label=containr.managed=true", "--filter", "label=containr.managed=true",
"--format", "{{.Names}}\t{{.State}}") "--format", "{{.Names}}\t{{.State}}")
if err != nil { if err != nil {
return nil return nil
} }
var states []map[string]string netIO := containerNetIO()
var states []map[string]interface{}
for _, line := range strings.Split(out, "\n") { for _, line := range strings.Split(out, "\n") {
parts := strings.SplitN(line, "\t", 2) parts := strings.SplitN(line, "\t", 2)
if len(parts) == 2 && parts[0] != "" { if len(parts) != 2 || parts[0] == "" {
states = append(states, map[string]string{"name": parts[0], "state": parts[1]}) continue
} }
entry := map[string]interface{}{"name": parts[0], "state": parts[1]}
if io, ok := netIO[parts[0]]; ok {
entry["rx_bytes"], entry["tx_bytes"] = io[0], io[1]
}
states = append(states, entry)
} }
return states return states
} }
// containerNetIO snapshots cumulative rx/tx per running container. Stats
// only covers running containers — stopped replicas report state without
// counters, which is all the sleeper needs.
func containerNetIO() map[string][2]int64 {
out, err := runDocker("stats", "--no-stream",
"--format", "{{.Name}}\t{{.NetIO}}")
if err != nil {
return nil
}
result := map[string][2]int64{}
for _, line := range strings.Split(out, "\n") {
parts := strings.SplitN(line, "\t", 2)
if len(parts) != 2 {
continue
}
io := strings.Split(parts[1], "/")
if len(io) != 2 {
continue
}
result[parts[0]] = [2]int64{parseHumanBytes(io[0]), parseHumanBytes(io[1])}
}
return result
}
// parseHumanBytes converts docker's human units ("1.2kB", "3.4MB") to bytes.
func parseHumanBytes(s string) int64 {
s = strings.TrimSpace(s)
var num float64
var unit string
if _, err := fmt.Sscanf(s, "%f%s", &num, &unit); err != nil {
return 0
}
switch strings.ToUpper(unit) {
case "B":
return int64(num)
case "KB":
return int64(num * 1e3)
case "MB":
return int64(num * 1e6)
case "GB":
return int64(num * 1e9)
case "TB":
return int64(num * 1e12)
case "KIB":
return int64(num * 1024)
case "MIB":
return int64(num * 1024 * 1024)
case "GIB":
return int64(num * 1024 * 1024 * 1024)
case "TIB":
return int64(num * 1024 * 1024 * 1024 * 1024)
}
return int64(num)
}
func processCommands(ctx context.Context, client *http.Client, cfg agentConfig, agentID string) error { func processCommands(ctx context.Context, client *http.Client, cfg agentConfig, agentID string) error {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, fmt.Sprintf("%s/api/agents/%s/commands", cfg.APIURL, agentID), nil) req, err := http.NewRequestWithContext(ctx, http.MethodGet, fmt.Sprintf("%s/api/agents/%s/commands", cfg.APIURL, agentID), nil)
if err != nil { if err != nil {
+4
View File
@@ -207,6 +207,8 @@ type AgentHeartbeat struct {
type AgentContainerState struct { type AgentContainerState struct {
Name string `json:"name"` Name string `json:"name"`
State string `json:"state"` State string `json:"state"`
RxBytes int64 `json:"rx_bytes,omitempty"`
TxBytes int64 `json:"tx_bytes,omitempty"`
} }
type SystemLoad struct { type SystemLoad struct {
@@ -681,10 +683,12 @@ func (h *NodeAgentHandler) SendHeartbeat(c *gin.Context) {
continue continue
} }
stateJSON, _ := json.Marshal(rc.State) stateJSON, _ := json.Marshal(rc.State)
bytesJSON, _ := json.Marshal(rc.RxBytes + rc.TxBytes)
_ = h.q.UpdateContainerStateByName(ctx, sqlcdb.UpdateContainerStateByNameParams{ _ = h.q.UpdateContainerStateByName(ctx, sqlcdb.UpdateContainerStateByNameParams{
Name: rc.Name, Name: rc.Name,
NodeAgentID: heartbeat.NodeAgentID, NodeAgentID: heartbeat.NodeAgentID,
Column3: stateJSON, Column3: stateJSON,
Column4: bytesJSON,
}) })
} }
+41
View File
@@ -169,3 +169,44 @@ func removeServiceIngress(serviceID string) {
} }
_ = os.Remove(ingressFile(serviceID)) _ = os.Remove(ingressFile(serviceID))
} }
// wakeUpstreamURL is where a sleeping remote service's route points —
// the backend's wake-page endpoint. Defaults to the compose service name;
// TRAEFIK_BACKEND_URL overrides for custom layouts.
func wakeUpstreamURL(apiPort int) string {
if v := strings.TrimSpace(os.Getenv("TRAEFIK_BACKEND_URL")); v != "" {
return strings.TrimRight(v, "/")
}
return fmt.Sprintf("http://containr-backend:%d", apiPort)
}
// writeWakeIngress routes the service's domains at the wake-page endpoint —
// first hit triggers the wake reconcile and serves a reloading page.
func writeWakeIngress(serviceID string, domains []string, apiPort int) {
if ingressDir() == "" {
return
}
sort.Strings(domains)
router := "svc-" + serviceID[:8] + "-wake"
rules := make([]string, 0, len(domains))
for _, d := range domains {
rules = append(rules, "Host(`"+d+"`)")
}
var b strings.Builder
mw := router + "-rw"
b.WriteString("http:\n routers:\n " + router + ":\n")
b.WriteString(" rule: \"" + strings.Join(rules, " || ") + "\"\n")
b.WriteString(" entrypoints: [web]\n")
b.WriteString(" service: " + router + "\n")
b.WriteString(" priority: 100\n")
b.WriteString(" middlewares: [" + mw + "]\n")
b.WriteString(" middlewares:\n " + mw + ":\n")
b.WriteString(" replacePath:\n path: \"/api/v1/internal/wake-page/" + serviceID + "\"\n")
b.WriteString(" services:\n " + router + ":\n loadBalancer:\n servers:\n")
b.WriteString(" - url: \"" + wakeUpstreamURL(apiPort) + "\"\n")
tmp := ingressFile(serviceID) + ".tmp"
if err := os.WriteFile(tmp, []byte(b.String()), 0o644); err != nil {
return
}
_ = os.Rename(tmp, ingressFile(serviceID))
}
+1
View File
@@ -285,6 +285,7 @@ func SetupRoutes(router *gin.Engine, db *database.DB, redis *database.Redis, cfg
// Wake callback for sleep-mode placeholders. Unauthenticated but // Wake callback for sleep-mode placeholders. Unauthenticated but
// non-destructive: it only reconciles the service back to running. // non-destructive: it only reconciles the service back to running.
public.Any("/internal/wake/:id", handleServiceWake) public.Any("/internal/wake/:id", handleServiceWake)
public.Any("/internal/wake-page/:id", handleServiceWakePage)
} }
// Read routes — public browsing. OptionalAuth resolves a session when // Read routes — public browsing. OptionalAuth resolves a session when
+107 -9
View File
@@ -13,6 +13,7 @@ import (
"time" "time"
"containr/internal/database" "containr/internal/database"
"containr/internal/database/sqlcdb"
"containr/internal/deployment" "containr/internal/deployment"
"containr/internal/docker" "containr/internal/docker"
@@ -100,7 +101,20 @@ func sweepSleeping(ctx context.Context, db *database.DB, dc *docker.Client, engi
} }
rows.Close() rows.Close()
q := sqlcdb.New(db.DB)
for _, svc := range candidates { for _, svc := range candidates {
// Remote replicas live in inventory, not the local docker host —
// idleness comes from heartbeat-reported net counters instead.
if remote := remoteLiveContainers(ctx, q, svc.ID.String()); len(remote) > 0 {
var total uint64
for _, row := range remote {
total += rowNetBytes(row)
}
if markIdle(svc.ID, total, time.Duration(svc.IdleMinutes)*time.Minute) {
putToSleepRemote(ctx, db, q, svc, apiPort)
}
continue
}
containers, err := engine.ListServiceContainers(ctx, svc.ID.String()) containers, err := engine.ListServiceContainers(ctx, svc.ID.String())
if err != nil || len(containers) == 0 { if err != nil || len(containers) == 0 {
continue continue
@@ -125,6 +139,65 @@ func sweepSleeping(ctx context.Context, db *database.DB, dc *docker.Client, engi
} }
} }
// remoteLiveContainers returns inventory rows whose replicas are alive on
// node agents — the signal that idleness must be measured remotely.
func remoteLiveContainers(ctx context.Context, q *sqlcdb.Queries, serviceID string) []sqlcdb.ListServiceContainersRow {
rows, err := q.ListServiceContainers(ctx, serviceID)
if err != nil {
return nil
}
live := rows[:0]
for _, row := range rows {
var st struct {
State string `json:"state"`
}
if err := json.Unmarshal(row.Status.RawMessage, &st); err == nil &&
st.State != "" && st.State != "removed" && st.State != "sleeping" {
live = append(live, row)
}
}
return live
}
// rowNetBytes reads the cumulative rx+tx the agent reported at its last
// heartbeat — 0 for rows written before the counter existed.
func rowNetBytes(row sqlcdb.ListServiceContainersRow) uint64 {
var res struct {
NetBytes uint64 `json:"net_bytes"`
}
_ = json.Unmarshal(row.Resources.RawMessage, &res)
return res.NetBytes
}
// putToSleepRemote removes the service's remote replicas via their agents,
// marks the service sleeping, and repoints the domain route at the wake
// endpoint so inbound traffic brings it back.
func putToSleepRemote(ctx context.Context, db *database.DB, q *sqlcdb.Queries, svc sleepCandidate, apiPort int) {
runner := newAgentNodeRunner(db)
for _, agentID := range remoteAgentsForService(ctx, db, svc.ID) {
if err := runner.ControlService(ctx, svc.ID.String(), agentID, "remove"); err != nil {
log.Printf("sleep: remote remove on %s failed: %v", agentID, err)
continue
}
_ = q.DeleteServiceContainersOnAgent(ctx, sqlcdb.DeleteServiceContainersOnAgentParams{
NodeAgentID: agentID,
ServiceID: svc.ID.String(),
})
}
tracker.mu.Lock()
delete(tracker.activity, svc.ID)
tracker.mu.Unlock()
_, _ = db.Exec(`UPDATE services SET status = 'sleeping', updated_at = $1 WHERE id = $2`, time.Now(), svc.ID)
log.Printf("sleep: %s (%s) remote replicas removed after idle timeout", svc.Name, svc.ID)
var domain string
_ = db.QueryRow(`SELECT COALESCE(domain,'') FROM services WHERE id = $1`, svc.ID).Scan(&domain)
domains := serviceDomainNames(db, svc.ID, domain)
if len(domains) > 0 {
writeWakeIngress(svc.ID.String(), domains, apiPort)
}
}
// markIdle returns true when the network counter stopped growing past the // markIdle returns true when the network counter stopped growing past the
// timeout. First sighting only sets the baseline — no idle without history. // timeout. First sighting only sets the baseline — no idle without history.
func markIdle(id uuid.UUID, bytes uint64, timeout time.Duration) bool { func markIdle(id uuid.UUID, bytes uint64, timeout time.Duration) bool {
@@ -346,18 +419,40 @@ func pruneWakeContainers(ctx context.Context, db *database.DB, dc *docker.Client
// idempotent and non-destructive, so no token is required; the endpoint // idempotent and non-destructive, so no token is required; the endpoint
// never mutates configuration. // never mutates configuration.
func handleServiceWake(c *gin.Context) { func handleServiceWake(c *gin.Context) {
code, body := triggerServiceWake(c)
if code != 0 {
c.JSON(code, body)
}
}
// handleServiceWakePage is the remote ingress wake target — Traefik routes
// a sleeping remote service's domain here via the file provider. Every hit
// (re)triggers the wake reconcile and returns a reloading page.
func handleServiceWakePage(c *gin.Context) {
triggerServiceWake(c)
c.Header("Content-Type", "text/html; charset=utf-8")
c.String(http.StatusOK, wakePageHTML)
}
const wakePageHTML = `<!doctype html><html><head><meta http-equiv="refresh" content="4"><title>Waking up</title>` +
`<style>body{font-family:system-ui,sans-serif;background:#0b0d12;color:#dde2ea;display:flex;align-items:center;justify-content:center;min-height:100vh;margin:0}` +
`.card{text-align:center;max-width:34rem;padding:2rem}h1{font-size:1.25rem;font-weight:600}p{color:#98a2b3}</style></head>` +
`<body><div class="card"><h1>Service is waking up</h1><p>This Containr service was sleeping and is resuming now. The page reloads automatically.</p></div></body></html>`
// triggerServiceWake loads the service and reconciles it back to running
// in the background. Returns (0, nil) when a response was already written
// by an embedded handler path, else the JSON status to send.
func triggerServiceWake(c *gin.Context) (int, gin.H) {
dbVal, _ := c.Get("db") dbVal, _ := c.Get("db")
db := dbVal.(*database.DB) db := dbVal.(*database.DB)
engineVal, _ := c.Get("deployment_engine") engineVal, _ := c.Get("deployment_engine")
engine, _ := engineVal.(*deployment.DeploymentEngine) engine, _ := engineVal.(*deployment.DeploymentEngine)
if engine == nil { if engine == nil {
c.JSON(http.StatusServiceUnavailable, gin.H{"error": "deployment engine unavailable"}) return http.StatusServiceUnavailable, gin.H{"error": "deployment engine unavailable"}
return
} }
id, err := uuid.Parse(c.Param("id")) id, err := uuid.Parse(c.Param("id"))
if err != nil { if err != nil {
c.JSON(http.StatusBadRequest, gin.H{"error": "invalid service id"}) return http.StatusBadRequest, gin.H{"error": "invalid service id"}
return
} }
var service Service var service Service
err = db.QueryRow( err = db.QueryRow(
@@ -385,12 +480,10 @@ func handleServiceWake(c *gin.Context) {
&service.CreatedAt, &service.UpdatedAt, &service.CreatedAt, &service.UpdatedAt,
) )
if err != nil { if err != nil {
c.JSON(http.StatusNotFound, gin.H{"error": "service not found"}) return http.StatusNotFound, gin.H{"error": "service not found"}
return
} }
if service.Status == "running" || service.Status == "deployed" { if service.Status == "running" || service.Status == "deployed" {
c.JSON(http.StatusOK, gin.H{"status": service.Status}) return http.StatusOK, gin.H{"status": service.Status}
return
} }
go func() { go func() {
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute) ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute)
@@ -411,7 +504,7 @@ func handleServiceWake(c *gin.Context) {
} }
_, _ = db.Exec(`UPDATE services SET status = $1, updated_at = $2 WHERE id = $3`, status, time.Now(), service.ID) _, _ = db.Exec(`UPDATE services SET status = $1, updated_at = $2 WHERE id = $3`, status, time.Now(), service.ID)
}() }()
c.JSON(http.StatusAccepted, gin.H{"status": "waking"}) return http.StatusAccepted, gin.H{"status": "waking"}
} }
// handleServiceSleep puts a service to sleep immediately (owner+ only). // handleServiceSleep puts a service to sleep immediately (owner+ only).
@@ -438,7 +531,12 @@ func handleServiceSleep(c *gin.Context) {
Name: service.Name, Name: service.Name,
Status: service.Status, Status: service.Status,
} }
q := sqlcdb.New(db.DB)
if len(remoteLiveContainers(c.Request.Context(), q, service.ID.String())) > 0 {
go putToSleepRemote(context.Background(), db, q, svc, apiPortFromContext(c))
} else {
go putToSleep(context.Background(), db, dc, engine, svc, apiPortFromContext(c)) go putToSleep(context.Background(), db, dc, engine, svc, apiPortFromContext(c))
}
c.JSON(http.StatusAccepted, gin.H{"status": "sleeping"}) c.JSON(http.StatusAccepted, gin.H{"status": "sleeping"})
} }
@@ -794,7 +794,7 @@ func (q *Queries) ListServiceAgents(ctx context.Context, serviceID string) ([]st
} }
const listServiceContainers = `-- name: ListServiceContainers :many const listServiceContainers = `-- name: ListServiceContainers :many
SELECT id, name, node_agent_id, status, ports FROM container_instances SELECT id, name, node_agent_id, status, ports, resources FROM container_instances
WHERE service_id = $1 WHERE service_id = $1
` `
@@ -804,6 +804,7 @@ type ListServiceContainersRow struct {
NodeAgentID string `json:"node_agent_id"` NodeAgentID string `json:"node_agent_id"`
Status pqtype.NullRawMessage `json:"status"` Status pqtype.NullRawMessage `json:"status"`
Ports pqtype.NullRawMessage `json:"ports"` Ports pqtype.NullRawMessage `json:"ports"`
Resources pqtype.NullRawMessage `json:"resources"`
} }
func (q *Queries) ListServiceContainers(ctx context.Context, serviceID string) ([]ListServiceContainersRow, error) { func (q *Queries) ListServiceContainers(ctx context.Context, serviceID string) ([]ListServiceContainersRow, error) {
@@ -821,6 +822,7 @@ func (q *Queries) ListServiceContainers(ctx context.Context, serviceID string) (
&i.NodeAgentID, &i.NodeAgentID,
&i.Status, &i.Status,
&i.Ports, &i.Ports,
&i.Resources,
); err != nil { ); err != nil {
return nil, err return nil, err
} }
@@ -1056,6 +1058,7 @@ func (q *Queries) UpdateContainerPorts(ctx context.Context, arg UpdateContainerP
const updateContainerStateByName = `-- name: UpdateContainerStateByName :exec const updateContainerStateByName = `-- name: UpdateContainerStateByName :exec
UPDATE container_instances SET UPDATE container_instances SET
status = jsonb_set(status, '{state}', $3::jsonb), status = jsonb_set(status, '{state}', $3::jsonb),
resources = jsonb_set(COALESCE(resources, '{}'::jsonb), '{net_bytes}', $4::jsonb),
updated_at = NOW() updated_at = NOW()
WHERE name = $1 AND node_agent_id = $2 WHERE name = $1 AND node_agent_id = $2
AND status->>'state' IS DISTINCT FROM 'removed' AND status->>'state' IS DISTINCT FROM 'removed'
@@ -1065,12 +1068,19 @@ type UpdateContainerStateByNameParams struct {
Name string `json:"name"` Name string `json:"name"`
NodeAgentID string `json:"node_agent_id"` NodeAgentID string `json:"node_agent_id"`
Column3 json.RawMessage `json:"column_3"` Column3 json.RawMessage `json:"column_3"`
Column4 json.RawMessage `json:"column_4"`
} }
// Agent heartbeat reports real docker state; reconcile inventory for rows // Agent heartbeat reports real docker state + net counters; reconcile
// this node owns. Tombstones win — never resurrect a removed replica. // inventory for rows this node owns. Tombstones win — never resurrect a
// removed replica. $4 is the cumulative rx+tx byte counter.
func (q *Queries) UpdateContainerStateByName(ctx context.Context, arg UpdateContainerStateByNameParams) error { func (q *Queries) UpdateContainerStateByName(ctx context.Context, arg UpdateContainerStateByNameParams) error {
_, err := q.db.ExecContext(ctx, updateContainerStateByName, arg.Name, arg.NodeAgentID, arg.Column3) _, err := q.db.ExecContext(ctx, updateContainerStateByName,
arg.Name,
arg.NodeAgentID,
arg.Column3,
arg.Column4,
)
return err return err
} }
@@ -139,8 +139,9 @@ type Querier interface {
// Agent reports the real host bindings after create — persist them with // Agent reports the real host bindings after create — persist them with
// the status flip so ingress can route to the assigned port. // the status flip so ingress can route to the assigned port.
UpdateContainerPorts(ctx context.Context, arg UpdateContainerPortsParams) error UpdateContainerPorts(ctx context.Context, arg UpdateContainerPortsParams) error
// Agent heartbeat reports real docker state; reconcile inventory for rows // Agent heartbeat reports real docker state + net counters; reconcile
// this node owns. Tombstones win — never resurrect a removed replica. // inventory for rows this node owns. Tombstones win — never resurrect a
// removed replica. $4 is the cumulative rx+tx byte counter.
UpdateContainerStateByName(ctx context.Context, arg UpdateContainerStateByNameParams) error UpdateContainerStateByName(ctx context.Context, arg UpdateContainerStateByNameParams) error
UpdateContainerStatus(ctx context.Context, arg UpdateContainerStatusParams) error UpdateContainerStatus(ctx context.Context, arg UpdateContainerStatusParams) error
UpdateDatabaseServiceNameAndPlanByIDAndUser(ctx context.Context, arg UpdateDatabaseServiceNameAndPlanByIDAndUserParams) error UpdateDatabaseServiceNameAndPlanByIDAndUser(ctx context.Context, arg UpdateDatabaseServiceNameAndPlanByIDAndUserParams) error
+5 -3
View File
@@ -180,7 +180,7 @@ LIMIT 1;
UPDATE node_agents SET tags = $2::jsonb, updated_at = NOW() WHERE id = $1; UPDATE node_agents SET tags = $2::jsonb, updated_at = NOW() WHERE id = $1;
-- name: ListServiceContainers :many -- name: ListServiceContainers :many
SELECT id, name, node_agent_id, status, ports FROM container_instances SELECT id, name, node_agent_id, status, ports, resources FROM container_instances
WHERE service_id = $1; WHERE service_id = $1;
-- name: SetAgentSchedulable :exec -- name: SetAgentSchedulable :exec
@@ -204,10 +204,12 @@ ON CONFLICT (id) DO UPDATE SET
DELETE FROM container_instances WHERE node_agent_id = $1 AND service_id = $2; DELETE FROM container_instances WHERE node_agent_id = $1 AND service_id = $2;
-- name: UpdateContainerStateByName :exec -- name: UpdateContainerStateByName :exec
-- Agent heartbeat reports real docker state; reconcile inventory for rows -- Agent heartbeat reports real docker state + net counters; reconcile
-- this node owns. Tombstones win — never resurrect a removed replica. -- inventory for rows this node owns. Tombstones win — never resurrect a
-- removed replica. $4 is the cumulative rx+tx byte counter.
UPDATE container_instances SET UPDATE container_instances SET
status = jsonb_set(status, '{state}', $3::jsonb), status = jsonb_set(status, '{state}', $3::jsonb),
resources = jsonb_set(COALESCE(resources, '{}'::jsonb), '{net_bytes}', $4::jsonb),
updated_at = NOW() updated_at = NOW()
WHERE name = $1 AND node_agent_id = $2 WHERE name = $1 AND node_agent_id = $2
AND status->>'state' IS DISTINCT FROM 'removed'; AND status->>'state' IS DISTINCT FROM 'removed';
+1 -1
View File
@@ -40,7 +40,7 @@ export const bundledDocs: BundledDoc[] = [
"path": "guides/CLI.md", "path": "guides/CLI.md",
"section": "guides", "section": "guides",
"title": "Containr CLI", "title": "Containr CLI",
"markdown": "# Containr CLI\n\nThe CLI controls every part of a Containr instance — the same API surface\nthe web UI uses. Build it with `scripts/build-cli.sh` or\n`go build ./cmd/cli` from `app/backend`.\n\n## Authentication\n\nCreate a personal access token in the web UI under\n**Settings → Personal Access Tokens** (or `containr tokens create`), then:\n\n```bash\ncontainr auth login cnp_... # verify + store\ncontainr auth login cnp_... --url https://containr.example.com\ncontainr auth login cnp_... --name production # named profile\ncontainr auth status # show active profile + identity\ncontainr auth logout\n```\n\nResolution order for every command: `--token`/`--api-url` flags >\n`CONTAINR_TOKEN`/`CONTAINR_API_URL` env > active profile >\n`http://localhost:8080/api/v1`. The active profile is `--profile` >\n`CONTAINR_PROFILE` > `current_profile` in `~/.containr.yaml`.\n\n## Output and scripting\n\n- `--json` prints the raw API response — safe to pipe and parse.\n- `--yes` / `-y` skips confirmation on destructive commands (delete,\n revoke, restore, failover). Non-interactive shells refuse destructive\n commands without it.\n- Exit codes are stable: `0` ok, `1` error, `2` auth/permission,\n `3` not found, `4` validation, `5` unreachable.\n\n## Commands\n\n```\nauth login|logout|status token auth and profiles\ntokens list|create|revoke personal access tokens\nprojects list|get|create|delete\nservices list|get|create|update|delete|start|stop|restart|redeploy\nservices clone|move duplicate config or re-parent a service\nservices domains list|add|remove|default|check multi-domain + DNS preflight\nregistries list|add|update|remove private-image pull credentials\nvolumes list|delete docker volume inventory + cleanup (admin)\ndeploy <service-id> trigger a deployment (--no-cache, --commit-hash, --branch)\ndeployments list|get|logs|rollback|cancel\nlogs <service-id> [--follow] runtime logs (SSE follow)\nexec <service-id> -- <cmd> one-off container command\nvariables list|set|unset env vars (secrets masked; --redeploy applies them)\nvariables shared list|set|unset project shared vars — referenced as ${{shared.KEY}}\ndatabases list|get|create|update|delete|action|backup|restore|download-backup|import-backup\n import-backup <id> <file.tar.gz|-> uploads an archive as a restore\n point; ships offsite too when a backup target is assigned\ndatabases list|get|create|update|delete|action|backup|restore|download-backup\n create --public exposes the port on all interfaces;\n update <id> --public/--public=false toggles it (container recreates, data persists)\ndatabases register <name> register an external database (--type --host --port --database --username --password --ssl; probed before storing)\ndatabases test-connection probe a connection — by id, or ad-hoc flags\nbackup-targets list|add|update|remove|test\n S3-compatible archive destinations; add/update probe the bucket\n first, empty --access-key/--secret-key on update keep stored keys;\n update <db-id> --backup-target assigns a target (\"\" clears it)\ncron list|get|create|delete|trigger|executions\ntemplates list|get|create|delete|deploy|plan|import-compose|deploy-graph\n deploy --name --var K=V; plan dry-resolves expressions;\n import-compose <compose.yml|-> converts a compose file into\n a v2 graph config; deploy-graph <project-id> <config.json|->\n deploys an ad-hoc service graph\nimport compose <project-id> <compose.yml|->\n deploys a docker-compose file straight into a project —\n no template row. DB images become managed databases\n attached under their service name; --var K=V overrides\n template variables\nnodes list|get|delete|tokens node agents + onboarding tokens (admin)\nnodes add <user@host> SSH bootstrap — one session installs the agent, then never needed again (--print for manual runs)\nnodes update|prune|commands rename/auto-prune, bounded docker prune, command history\nmigrate railway pull a Railway project (services, vars, domains) via GraphQL and recreate it — --dry-run prints the plan\nmigrate dflow import a dflow project (services, vars, volumes, domains) via its Payload REST API — needs --url + --api-key, --dry-run prints the plan\nscaling status|policies|services|scale\nha status|alerts|health|policies|enable|disable|failover\nsecurity scan|vulnerabilities|metrics\ngateway services|keys|anLine truncated "markdown": "# Containr CLI\n\nThe CLI controls every part of a Containr instance — the same API surface\nthe web UI uses. Build it with `scripts/build-cli.sh` or\n`go build ./cmd/cli` from `app/backend`.\n\n## Authentication\n\nCreate a personal access token in the web UI under\n**Settings → Personal Access Tokens** (or `containr tokens create`), then:\n\n```bash\ncontainr auth login cnp_... # verify + store\ncontainr auth login cnp_... --url https://containr.example.com\ncontainr auth login cnp_... --name production # named profile\ncontainr auth status # show active profile + identity\ncontainr auth logout\n```\n\nResolution order for every command: `--token`/`--api-url` flags >\n`CONTAINR_TOKEN`/`CONTAINR_API_URL` env > active profile >\n`http://localhost:8080/api/v1`. The active profile is `--profile` >\n`CONTAINR_PROFILE` > `current_profile` in `~/.containr.yaml`.\n\n## Output and scripting\n\n- `--json` prints the raw API response — safe to pipe and parse.\n- `--yes` / `-y` skips confirmation on destructive commands (delete,\n revoke, restore, failover). Non-interactive shells refuse destructive\n commands without it.\n- Exit codes are stable: `0` ok, `1` error, `2` auth/permission,\n `3` not found, `4` validation, `5` unreachable.\n\n## Commands\n\n```\nauth login|logout|status token auth and profiles\ntokens list|create|revoke personal access tokens\nprojects list|get|create|delete\nservices list|get|create|update|delete|start|stop|restart|redeploy\nservices clone|move duplicate config or re-parent a service\nservices domains list|add|remove|default|check multi-domain + DNS preflight\nregistries list|add|update|remove private-image pull credentials\nvolumes list|delete docker volume inventory + cleanup (admin)\ndeploy <service-id> trigger a deployment (--no-cache, --commit-hash, --branch)\ndeployments list|get|logs|rollback|cancel\nlogs <service-id> [--follow] runtime logs (SSE follow)\nexec <service-id> -- <cmd> one-off container command\nvariables list|set|unset env vars (secrets masked; --redeploy applies them)\nvariables shared list|set|unset project shared vars — referenced as ${{shared.KEY}}\ndatabases list|get|create|update|delete|action|backup|restore|download-backup|import-backup\n import-backup <id> <file.tar.gz|-> uploads an archive as a restore\n point; ships offsite too when a backup target is assigned\ndatabases list|get|create|update|delete|action|backup|restore|download-backup\n create --public exposes the port on all interfaces;\n update <id> --public/--public=false toggles it (container recreates, data persists)\ndatabases register <name> register an external database (--type --host --port --database --username --password --ssl; probed before storing)\ndatabases test-connection probe a connection — by id, or ad-hoc flags\nbackup-targets list|add|update|remove|test\n S3-compatible archive destinations; add/update probe the bucket\n first, empty --access-key/--secret-key on update keep stored keys;\n update <db-id> --backup-target assigns a target (\"\" clears it)\ncron list|get|create|delete|trigger|executions\ntemplates list|get|create|delete|deploy|plan|import-compose|deploy-graph\n deploy --name --var K=V; plan dry-resolves expressions;\n import-compose <compose.yml|-> converts a compose file into\n a v2 graph config; deploy-graph <project-id> <config.json|->\n deploys an ad-hoc service graph\nimport compose <project-id> <compose.yml|->\n deploys a docker-compose file straight into a project —\n no template row. DB images become managed databases\n attached under their service name; --var K=V overrides\n template variables\nnodes list|get|delete|tokens node agents + onboarding tokens (admin)\nnodes add <user@host> SSH bootstrap — one session installs the agent, then never needed again (--print for manual runs)\nnodes update|prune|commands rename/auto-prune, bounded docker prune, command history\nmigrate railway pull a Railway project (services, vars, domains) via GraphQL and recreate it — --dry-run prints the plan\nmigrate dflow import a dflow project (services, vars, volumes, domains) via its Payload REST API — needs --url + --api-key, --dry-run prints the plan\nscaling status|policies|services|scale\nha status|alerts|health|policies|enable|disable|failover\nsecurity scan|vulnerabilities|metrics\ngateway services|keys|anLine truncated
}, },
{ {
"path": "guides/CLOUDFLARE_SETUP.md", "path": "guides/CLOUDFLARE_SETUP.md",
+9 -6
View File
@@ -161,12 +161,15 @@ containr nodes update <id> --tags gpu,eu-west
containr services update <id> --placement-tags gpu # only gpu nodes are eligible containr services update <id> --placement-tags gpu # only gpu nodes are eligible
``` ```
Remote nodes run registry-pulled images only — git builds and sleep Remote nodes run registry-pulled images only — git builds stay local.
detection stay local. Domains work remotely: the agent publishes the Domains work remotely: the agent publishes the service port and the
service port and the backend writes a Traefik file-provider route to the backend writes a Traefik file-provider route to the node's mesh IP
node's mesh IP (TRAEFIK_DYNAMIC_DIR — wired by the infra compose). (TRAEFIK_DYNAMIC_DIR — wired by the infra compose). Sleep mode works
Start, stop, restart, redeploy, and delete all fan out to the pinned too: heartbeats carry per-container net counters, idle remote services
node's agent. `services get` shows the resolved `node_name`. lose their replicas via the agent, and the route repoints at the
backend's wake page — the next hit reconciles the service back. Start,
stop, restart, redeploy, and delete all fan out to the pinned node's
agent. `services get` shows the resolved `node_name`.
Take a node out of rotation or empty it entirely (admin): Take a node out of rotation or empty it entirely (admin):
+1
View File
@@ -103,6 +103,7 @@ services:
- CONTAINR_AGENT_AUTH_TOKEN=${CONTAINR_AGENT_AUTH_TOKEN:-} - CONTAINR_AGENT_AUTH_TOKEN=${CONTAINR_AGENT_AUTH_TOKEN:-}
- CONTAINR_AGENT_AUTH_TOKENS=${CONTAINR_AGENT_AUTH_TOKENS:-} - CONTAINR_AGENT_AUTH_TOKENS=${CONTAINR_AGENT_AUTH_TOKENS:-}
- TRAEFIK_DYNAMIC_DIR=/traefik-dynamic - TRAEFIK_DYNAMIC_DIR=/traefik-dynamic
- TRAEFIK_BACKEND_URL=http://containr-backend:8080
- COOKIE_SECURE=${COOKIE_SECURE:-true} - COOKIE_SECURE=${COOKIE_SECURE:-true}
- TRUSTED_PROXY_CIDR=${TRUSTED_PROXY_CIDR:-172.20.0.0/16} - TRUSTED_PROXY_CIDR=${TRUSTED_PROXY_CIDR:-172.20.0.0/16}
- CORS_ORIGINS=${CORS_ORIGINS:-http://localhost,http://localhost:3000} - CORS_ORIGINS=${CORS_ORIGINS:-http://localhost,http://localhost:3000}