diff --git a/.env.example b/.env.example index 441f421..223a5b7 100644 --- a/.env.example +++ b/.env.example @@ -49,6 +49,10 @@ CONTAINR_AGENT_AUTH_TOKEN=replace_with_strong_agent_secret # CONTAINR_AGENT_AUTH_TOKENS=current_secret,next_secret # Bundled agent binaries dir served at /api/agents/download/:platform. # CONTAINR_AGENT_BIN_DIR=/agents +# Directory Traefik's file provider watches. The backend writes one +# config per remote service here so domains route to node agents. The +# infra compose wires a shared volume — set only for custom layouts. +# TRAEFIK_DYNAMIC_DIR=/traefik-dynamic # Optional remote template catalog (another Containr instance). Its # templates merge into GET /templates as source:"registry" and # materialize locally on deploy. The bundled catalog works without it. diff --git a/ROADMAP2.md b/ROADMAP2.md index b5006de..d6b5a22 100644 --- a/ROADMAP2.md +++ b/ROADMAP2.md @@ -296,9 +296,22 @@ and wired refs, one click or one CLI call. utilisation, then cpu, then container count, with uninstrumented agents sorting last. Lifecycle and `GET /runtime` are inventory-driven, so remote replicas show up and tear down wherever - they run. Remote nodes run registry-pulled - images only — git builds, Traefik domains, and remote sleep are - documented ceilings. Tag affinity shipped: `node_agents.tags` + they run. Remote ingress shipped: the agent reports assigned host + ports in the `create_container` result, the backend renders a + Traefik file-provider config per remote service into + `TRAEFIK_DYNAMIC_DIR` (shared volume, `watch=true`) pointing the + loadbalancer at each replica's mesh IP + published port; route files + are rewritten on every remote reconcile and removed on teardown or + local placement. Agents report managed-container states at every + heartbeat — exits, OOMs, and crash loops land in inventory between + reconciles (tombstones never resurrect). Remote sleep/wake shipped: + heartbeats carry per-container net counters, idle remote services + lose replicas via agent remove + tombstones, and the route file + repoints at `/internal/wake-page/:id` on the backend — first hit + retriggers the wake reconcile, which recreates replicas and restores + real upstreams. Remote nodes run + registry-pulled images only — git builds are the remaining + documented ceiling. Tag affinity shipped: `node_agents.tags` (operator-set via `PUT /agents/:id` / `nodes update --tags`) + `services.placement_tags` — `auto` and `spread` only consider online schedulable agents carrying every required tag, tags are @@ -307,9 +320,8 @@ and wired refs, one click or one CLI call. and local + tags are rejected, empty array clears. Surfaces: `--placement-tags` on services create/update, `--tags` on `nodes update`, MCP args, OpenAPI, node-detail tag editor + - service-detail placement-tags input. Remaining: deployment - status reported back from agent results, spread weighting by - real capacity. + service-detail placement-tags input. Remaining: build-on-node, + remote sleep/wake, spread weighting by real capacity. - [x] **Agent self-upgrade** — `POST /agents/:id/upgrade` enqueues a `self_upgrade` command carrying platform + sha256 + server version; the agent downloads its replacement from `GET /agents/download/ diff --git a/app/backend/cmd/agent/main.go b/app/backend/cmd/agent/main.go index 48d2124..34d4ed4 100644 --- a/app/backend/cmd/agent/main.go +++ b/app/backend/cmd/agent/main.go @@ -204,10 +204,92 @@ func sendHeartbeat(ctx context.Context, client *http.Client, cfg agentConfig, ag "system_load": readSystemLoad(), "uptime": readSystemUptime(), "version": firstEnv("CONTAINR_AGENT_VERSION", version), + "containers": managedContainerStates(), } return postJSON(ctx, client, cfg, "/api/agents/heartbeat", payload, nil) } +// managedContainerStates lists every containr-managed container's docker +// state plus cumulative network counters — state keeps runtime honest, +// net bytes let the sleeper detect idle remote services. +func managedContainerStates() []map[string]interface{} { + out, err := runDocker("ps", "-a", + "--filter", "label=containr.managed=true", + "--format", "{{.Names}}\t{{.State}}") + if err != nil { + return nil + } + netIO := containerNetIO() + var states []map[string]interface{} + for _, line := range strings.Split(out, "\n") { + parts := strings.SplitN(line, "\t", 2) + if len(parts) != 2 || parts[0] == "" { + continue + } + entry := map[string]interface{}{"name": parts[0], "state": parts[1]} + if io, ok := netIO[parts[0]]; ok { + entry["rx_bytes"], entry["tx_bytes"] = io[0], io[1] + } + states = append(states, entry) + } + return states +} + +// containerNetIO snapshots cumulative rx/tx per running container. Stats +// only covers running containers — stopped replicas report state without +// counters, which is all the sleeper needs. +func containerNetIO() map[string][2]int64 { + out, err := runDocker("stats", "--no-stream", + "--format", "{{.Name}}\t{{.NetIO}}") + if err != nil { + return nil + } + result := map[string][2]int64{} + for _, line := range strings.Split(out, "\n") { + parts := strings.SplitN(line, "\t", 2) + if len(parts) != 2 { + continue + } + io := strings.Split(parts[1], "/") + if len(io) != 2 { + continue + } + result[parts[0]] = [2]int64{parseHumanBytes(io[0]), parseHumanBytes(io[1])} + } + return result +} + +// parseHumanBytes converts docker's human units ("1.2kB", "3.4MB") to bytes. +func parseHumanBytes(s string) int64 { + s = strings.TrimSpace(s) + var num float64 + var unit string + if _, err := fmt.Sscanf(s, "%f%s", &num, &unit); err != nil { + return 0 + } + switch strings.ToUpper(unit) { + case "B": + return int64(num) + case "KB": + return int64(num * 1e3) + case "MB": + return int64(num * 1e6) + case "GB": + return int64(num * 1e9) + case "TB": + return int64(num * 1e12) + case "KIB": + return int64(num * 1024) + case "MIB": + return int64(num * 1024 * 1024) + case "GIB": + return int64(num * 1024 * 1024 * 1024) + case "TIB": + return int64(num * 1024 * 1024 * 1024 * 1024) + } + return int64(num) +} + func processCommands(ctx context.Context, client *http.Client, cfg agentConfig, agentID string) error { req, err := http.NewRequestWithContext(ctx, http.MethodGet, fmt.Sprintf("%s/api/agents/%s/commands", cfg.APIURL, agentID), nil) if err != nil { @@ -272,7 +354,12 @@ func executeCommand(item command) (string, error) { case "restart_container": return runDocker("restart", commandTarget(item.Payload)) case "remove_container": - return runDocker("rm", "-f", commandTarget(item.Payload)) + out, err := runDocker("rm", "-f", commandTarget(item.Payload)) + // A missing container is the desired end state — report success. + if err != nil && strings.Contains(out, "No such container") { + return out, nil + } + return out, err case "prune": return dockerPrune(item.Payload) case "system_df": @@ -460,7 +547,25 @@ func createContainer(payload map[string]interface{}) (string, error) { logs, _ := runDocker("logs", "--tail", "5", name) return "", fmt.Errorf("container %s is %q after start: %s", name, state, logs) } - return name, nil + // Report the real host port bindings — ephemeral publishes pick a port + // the server can't guess, and remote ingress needs the actual mapping. + hostPorts := map[string]string{} + portsJSON, _ := runDocker("inspect", "-f", "{{json .NetworkSettings.Ports}}", name) + var bindings map[string][]struct { + HostIP string `json:"HostIp"` + HostPort string `json:"HostPort"` + } + if json.Unmarshal([]byte(portsJSON), &bindings) == nil { + for cport, addrs := range bindings { + for _, a := range addrs { + if a.HostPort != "" && a.HostPort != "0" { + hostPorts[cport] = a.HostPort + } + } + } + } + out, _ := json.Marshal(map[string]interface{}{"name": name, "host_ports": hostPorts}) + return string(out), nil } // registryHostOf mirrors the server's registryHost: first path segment with diff --git a/app/backend/internal/api/agents.go b/app/backend/internal/api/agents.go index b7bc75b..295008b 100644 --- a/app/backend/internal/api/agents.go +++ b/app/backend/internal/api/agents.go @@ -198,6 +198,17 @@ type AgentHeartbeat struct { SystemLoad SystemLoad `json:"system_load"` Uptime int64 `json:"uptime"` Version string `json:"version"` + // Containers reports managed-container liveness — {name, state} per + // containr.managed container on the node — so remote replica status + // reflects reality between reconciles (exits, OOMs, crash loops). + Containers []AgentContainerState `json:"containers,omitempty"` +} + +type AgentContainerState struct { + Name string `json:"name"` + State string `json:"state"` + RxBytes int64 `json:"rx_bytes,omitempty"` + TxBytes int64 `json:"tx_bytes,omitempty"` } type SystemLoad struct { @@ -665,6 +676,22 @@ func (h *NodeAgentHandler) SendHeartbeat(c *gin.Context) { return } + // Reconcile remote container liveness — exits and crash loops surface + // in inventory between reconciles. Best-effort, never fatal. + for _, rc := range heartbeat.Containers { + if rc.Name == "" || rc.State == "" { + continue + } + stateJSON, _ := json.Marshal(rc.State) + bytesJSON, _ := json.Marshal(rc.RxBytes + rc.TxBytes) + _ = h.q.UpdateContainerStateByName(ctx, sqlcdb.UpdateContainerStateByNameParams{ + Name: rc.Name, + NodeAgentID: heartbeat.NodeAgentID, + Column3: stateJSON, + Column4: bytesJSON, + }) + } + c.Status(http.StatusOK) } diff --git a/app/backend/internal/api/ingress.go b/app/backend/internal/api/ingress.go new file mode 100644 index 0000000..31a278d --- /dev/null +++ b/app/backend/internal/api/ingress.go @@ -0,0 +1,212 @@ +package api + +import ( + "context" + "encoding/json" + "fmt" + "os" + "path/filepath" + "sort" + "strings" + + "containr/internal/database" + "containr/internal/database/sqlcdb" + "containr/internal/deployment" +) + +// Remote ingress: replicas on node agents aren't on the hub's +// containr-network, so Traefik's Docker provider can't see them. Instead +// the backend renders one file-provider config per service into +// TRAEFIK_DYNAMIC_DIR (a directory Traefik watches), pointing the +// loadbalancer at the node's reachable IP + the ephemeral host port the +// agent reported back after `create_container`. +// +// Empty dir → feature off (bare-metal dev). Files are best-effort: a +// failed write never fails a reconcile. + +func ingressDir() string { return os.Getenv("TRAEFIK_DYNAMIC_DIR") } + +func ingressFile(serviceID string) string { + return filepath.Join(ingressDir(), "containr-"+serviceID+".yml") +} + +// jsonInt coerces JSON-ish numbers (int/int32/int64/float64) to int — +// spec fields arrive as int32, agent payloads unmarshal as float64. +func jsonInt(v interface{}) int { + switch n := v.(type) { + case int: + return n + case int32: + return int(n) + case int64: + return int(n) + case float64: + return int(n) + } + return 0 +} + +// reportedHostPorts parses the agent's create_container result — +// {"name": ..., "host_ports": {"80/tcp": "32768"}}. Older agents return a +// bare name string; JSON parse failure means no port info. +func reportedHostPorts(result string) map[string]string { + var out struct { + HostPorts map[string]string `json:"host_ports"` + } + if json.Unmarshal([]byte(result), &out) != nil { + return nil + } + return out.HostPorts +} + +// nodeIngressIP picks the address the hub can reach the node on. Mesh +// overlays first — the agent operator joined a mesh precisely so the hub +// can reach it — then the reported primary IP as a LAN fallback. +func nodeIngressIP(agent sqlcdb.NodeAgent) string { + var meta struct { + Mesh map[string]string `json:"mesh"` + } + _ = json.Unmarshal(agent.Metadata.RawMessage, &meta) + for _, prefix := range []string{"tailscale", "netbird", "wg", "zt"} { + var keys []string + for name := range meta.Mesh { + keys = append(keys, name) + } + sort.Strings(keys) + for _, name := range keys { + if strings.HasPrefix(name, prefix) && meta.Mesh[name] != "" { + return meta.Mesh[name] + } + } + } + return agent.IpAddress +} + +// syncRemoteIngress rewrites the service's file-provider config from the +// live remote replica set. Called after every remote reconcile so replica +// adds/removals and node retargets converge the upstream list. +func syncRemoteIngress(ctx context.Context, db *database.DB, spec deployment.RuntimeSpec) { + if ingressDir() == "" { + return + } + domains := spec.Domains + if len(domains) == 0 && spec.Domain != "" { + domains = []string{spec.Domain} + } + if len(domains) == 0 || spec.Port <= 0 { + removeServiceIngress(spec.ServiceID) + return + } + + q := sqlcdb.New(db.DB) + rows, err := q.ListServiceContainers(ctx, spec.ServiceID) + if err != nil { + return + } + type upstream struct{ ip, port string } + var servers []upstream + for _, row := range rows { + var st struct { + State string `json:"state"` + } + _ = json.Unmarshal(row.Status.RawMessage, &st) + if st.State != "running" { + continue + } + var port string + var plist []map[string]interface{} + if err := json.Unmarshal(row.Ports.RawMessage, &plist); err == nil { + for _, p := range plist { + if hp, ok := p["host_port"].(float64); ok && hp > 0 { + port = fmt.Sprintf("%d", int(hp)) + } + } + } + if port == "" { + continue + } + agent, err := q.GetAgent(ctx, row.NodeAgentID) + if err != nil { + continue + } + if ip := nodeIngressIP(agent); ip != "" { + servers = append(servers, upstream{ip, port}) + } + } + if len(servers) == 0 { + removeServiceIngress(spec.ServiceID) + return + } + + router := "svc-" + spec.ServiceID[:8] + rules := make([]string, 0, len(domains)) + for _, d := range domains { + rules = append(rules, "Host(`"+d+"`)") + } + var b strings.Builder + b.WriteString("http:\n routers:\n " + router + ":\n") + b.WriteString(" rule: \"" + strings.Join(rules, " || ") + "\"\n") + b.WriteString(" entrypoints: [web]\n") + b.WriteString(" service: " + router + "\n") + b.WriteString(" services:\n " + router + ":\n loadBalancer:\n servers:\n") + for _, s := range servers { + b.WriteString(" - url: \"http://" + s.ip + ":" + s.port + "\"\n") + } + // Atomic-ish: write temp then rename so Traefik's watcher never reads a + // truncated file. + tmp := ingressFile(spec.ServiceID) + ".tmp" + if err := os.WriteFile(tmp, []byte(b.String()), 0o644); err != nil { + return + } + _ = os.Rename(tmp, ingressFile(spec.ServiceID)) +} + +// removeServiceIngress deletes the service's remote route. Called on +// remote teardown and when a service returns to local placement. +func removeServiceIngress(serviceID string) { + if ingressDir() == "" { + return + } + _ = os.Remove(ingressFile(serviceID)) +} + +// wakeUpstreamURL is where a sleeping remote service's route points — +// the backend's wake-page endpoint. Defaults to the compose service name; +// TRAEFIK_BACKEND_URL overrides for custom layouts. +func wakeUpstreamURL(apiPort int) string { + if v := strings.TrimSpace(os.Getenv("TRAEFIK_BACKEND_URL")); v != "" { + return strings.TrimRight(v, "/") + } + return fmt.Sprintf("http://containr-backend:%d", apiPort) +} + +// writeWakeIngress routes the service's domains at the wake-page endpoint — +// first hit triggers the wake reconcile and serves a reloading page. +func writeWakeIngress(serviceID string, domains []string, apiPort int) { + if ingressDir() == "" { + return + } + sort.Strings(domains) + router := "svc-" + serviceID[:8] + "-wake" + rules := make([]string, 0, len(domains)) + for _, d := range domains { + rules = append(rules, "Host(`"+d+"`)") + } + var b strings.Builder + mw := router + "-rw" + b.WriteString("http:\n routers:\n " + router + ":\n") + b.WriteString(" rule: \"" + strings.Join(rules, " || ") + "\"\n") + b.WriteString(" entrypoints: [web]\n") + b.WriteString(" service: " + router + "\n") + b.WriteString(" priority: 100\n") + b.WriteString(" middlewares: [" + mw + "]\n") + b.WriteString(" middlewares:\n " + mw + ":\n") + b.WriteString(" replacePath:\n path: \"/api/v1/internal/wake-page/" + serviceID + "\"\n") + b.WriteString(" services:\n " + router + ":\n loadBalancer:\n servers:\n") + b.WriteString(" - url: \"" + wakeUpstreamURL(apiPort) + "\"\n") + tmp := ingressFile(serviceID) + ".tmp" + if err := os.WriteFile(tmp, []byte(b.String()), 0o644); err != nil { + return + } + _ = os.Rename(tmp, ingressFile(serviceID)) +} diff --git a/app/backend/internal/api/noderunner.go b/app/backend/internal/api/noderunner.go index 7717596..1f181bb 100644 --- a/app/backend/internal/api/noderunner.go +++ b/app/backend/internal/api/noderunner.go @@ -19,8 +19,8 @@ import ( // agentNodeRunner implements deployment.NodeRunner by dispatching container // lifecycle commands to a node agent's command queue and waiting for the -// reported result. Remote nodes have no Traefik — domains stay local-only -// until per-node ingress lands. +// reported result. Domains route through the file provider — +// syncRemoteIngress renders per-service configs from live inventory. type agentNodeRunner struct { q *sqlcdb.Queries db *database.DB @@ -131,10 +131,6 @@ func (r *agentNodeRunner) RemoteRuntimeState(ctx context.Context, serviceID stri // targetFor and retires stale replicas on any agent — including leftovers // from a previous pin or a different spread assignment. func (r *agentNodeRunner) reconcileOnAgents(ctx context.Context, spec deployment.RuntimeSpec, targetFor func(replica int) string) (*deployment.RuntimeState, error) { - if len(spec.Domains) > 0 || spec.Domain != "" { - return nil, fmt.Errorf("domains are routed by the local Traefik — remove the service's domains or run it on the local node") - } - replicas := spec.Replicas if replicas < 1 { replicas = 1 @@ -151,7 +147,7 @@ func (r *agentNodeRunner) reconcileOnAgents(ctx context.Context, spec deployment if keep { continue } - if cmdErr := r.enqueueAndWait(ctx, row.NodeAgentID, "", "remove_container", + if _, cmdErr := r.enqueueAndWait(ctx, row.NodeAgentID, "", "remove_container", map[string]interface{}{"container_name": row.Name}); cmdErr != nil { return nil, fmt.Errorf("remove stale replica %s on node %s: %w", row.Name, row.NodeAgentID, cmdErr) } @@ -185,18 +181,41 @@ func (r *agentNodeRunner) reconcileOnAgents(ctx context.Context, spec deployment HealthCheck: rawJSON(map[string]interface{}{"path": spec.HealthPath}), }) - if err := r.enqueueAndWait(ctx, agentID, containerID, "create_container", - map[string]interface{}{"container": container}); err != nil { + result, err := r.enqueueAndWait(ctx, agentID, containerID, "create_container", + map[string]interface{}{"container": container}) + if err != nil { return nil, fmt.Errorf("replica %d on node %s: %w", i, agentID, err) } - _ = r.q.UpdateContainerStatus(ctx, sqlcdb.UpdateContainerStatusParams{ - ID: containerID, - Status: rawJSON(map[string]interface{}{"state": "running", "health": "none"}), - }) + if assigned := reportedHostPorts(result); len(assigned) > 0 { + for _, p := range ports { + pm, ok := p.(map[string]interface{}) + if !ok { + continue + } + cport := jsonInt(pm["container_port"]) + proto, _ := pm["protocol"].(string) + if hp, ok := assigned[fmt.Sprintf("%d/%s", cport, proto)]; ok { + if n, err := strconv.Atoi(hp); err == nil { + pm["host_port"] = n + } + } + } + _ = r.q.UpdateContainerPorts(ctx, sqlcdb.UpdateContainerPortsParams{ + ID: containerID, + Status: rawJSON(map[string]interface{}{"state": "running", "health": "none"}), + Ports: rawJSON(ports), + }) + } else { + _ = r.q.UpdateContainerStatus(ctx, sqlcdb.UpdateContainerStatusParams{ + ID: containerID, + Status: rawJSON(map[string]interface{}{"state": "running", "health": "none"}), + }) + } state.Containers = append(state.Containers, deployment.RuntimeContainer{ ID: containerID, Name: name, State: "running", Replica: i, }) } + syncRemoteIngress(ctx, r.db, spec) return state, nil } @@ -296,7 +315,17 @@ func (r *agentNodeRunner) ControlService(ctx context.Context, serviceID, agentID if row.ServiceID != serviceID { continue } - if err := r.enqueueAndWait(ctx, agentID, row.ID, cmdType, + // Tombstoned replicas are already gone remotely — skip the dispatch + // or a fresh docker rm on a missing name reports a false failure. + if action == "remove" { + var st struct { + State string `json:"state"` + } + if err := json.Unmarshal(row.Status.RawMessage, &st); err == nil && st.State == "removed" { + continue + } + } + if _, err := r.enqueueAndWait(ctx, agentID, row.ID, cmdType, map[string]interface{}{"container_name": row.Name}); err != nil { return fmt.Errorf("%s %s on node: %w", action, row.Name, err) } @@ -314,16 +343,38 @@ func (r *agentNodeRunner) RemoveService(ctx context.Context, serviceID, agentID if err := r.ControlService(ctx, serviceID, agentID, "remove"); err != nil { return err } - return r.q.DeleteServiceContainersOnAgent(ctx, sqlcdb.DeleteServiceContainersOnAgentParams{ + if err := r.q.DeleteServiceContainersOnAgent(ctx, sqlcdb.DeleteServiceContainersOnAgentParams{ NodeAgentID: agentID, ServiceID: serviceID, - }) + }); err != nil { + return err + } + removeServiceIngress(serviceID) + return nil +} + +// RetireService removes the service's remote replicas wherever they run — +// called when the service converges to local placement so a cleared pin or +// removed tag requirement doesn't leave containers behind. +func (r *agentNodeRunner) RetireService(ctx context.Context, serviceID string) error { + agentIDs, err := r.q.ListServiceAgents(ctx, serviceID) + if err != nil { + return err + } + for _, agentID := range agentIDs { + if err := r.RemoveService(ctx, serviceID, agentID); err != nil { + return fmt.Errorf("retire on node %s: %w", agentID, err) + } + } + removeServiceIngress(serviceID) + return nil } // enqueueAndWait pushes a command to the agent's queue and polls for the -// reported result. Agents poll on their own interval (≥5s), so this blocks — -// it runs inside the deploy queue worker, never on a request path. -func (r *agentNodeRunner) enqueueAndWait(ctx context.Context, agentID, containerID, cmdType string, payload map[string]interface{}) error { +// reported result, returning the agent's result string on success. Agents +// poll on their own interval (≥5s), so this blocks — it runs inside the +// deploy queue worker, never on a request path. +func (r *agentNodeRunner) enqueueAndWait(ctx context.Context, agentID, containerID, cmdType string, payload map[string]interface{}) (string, error) { var cid sql.NullString if containerID != "" { cid = sql.NullString{String: containerID, Valid: true} @@ -336,7 +387,7 @@ func (r *agentNodeRunner) enqueueAndWait(ctx context.Context, agentID, container Payload: rawJSON(payload), }) if err != nil { - return err + return "", err } deadline := time.Now().Add(8 * time.Minute) // image pulls on a cold node @@ -348,20 +399,20 @@ func (r *agentNodeRunner) enqueueAndWait(ctx context.Context, agentID, container if err == nil { switch row.Status.String { case "completed": - return nil + return row.Result.String, nil case "failed": if row.Error.Valid && row.Error.String != "" { - return fmt.Errorf("agent: %s", row.Error.String) + return "", fmt.Errorf("agent: %s", row.Error.String) } - return fmt.Errorf("agent reported failure") + return "", fmt.Errorf("agent reported failure") } } if time.Now().After(deadline) { - return fmt.Errorf("command %s timed out waiting for the agent", cmdType) + return "", fmt.Errorf("command %s timed out waiting for the agent", cmdType) } select { case <-ctx.Done(): - return ctx.Err() + return "", ctx.Err() case <-time.After(2 * time.Second): } } diff --git a/app/backend/internal/api/routes.go b/app/backend/internal/api/routes.go index f7ff060..318d60c 100644 --- a/app/backend/internal/api/routes.go +++ b/app/backend/internal/api/routes.go @@ -285,6 +285,7 @@ func SetupRoutes(router *gin.Engine, db *database.DB, redis *database.Redis, cfg // Wake callback for sleep-mode placeholders. Unauthenticated but // non-destructive: it only reconciles the service back to running. public.Any("/internal/wake/:id", handleServiceWake) + public.Any("/internal/wake-page/:id", handleServiceWakePage) } // Read routes — public browsing. OptionalAuth resolves a session when diff --git a/app/backend/internal/api/sleeper.go b/app/backend/internal/api/sleeper.go index 4e4f7f4..66198ee 100644 --- a/app/backend/internal/api/sleeper.go +++ b/app/backend/internal/api/sleeper.go @@ -13,6 +13,7 @@ import ( "time" "containr/internal/database" + "containr/internal/database/sqlcdb" "containr/internal/deployment" "containr/internal/docker" @@ -100,7 +101,20 @@ func sweepSleeping(ctx context.Context, db *database.DB, dc *docker.Client, engi } rows.Close() + q := sqlcdb.New(db.DB) for _, svc := range candidates { + // Remote replicas live in inventory, not the local docker host — + // idleness comes from heartbeat-reported net counters instead. + if remote := remoteLiveContainers(ctx, q, svc.ID.String()); len(remote) > 0 { + var total uint64 + for _, row := range remote { + total += rowNetBytes(row) + } + if markIdle(svc.ID, total, time.Duration(svc.IdleMinutes)*time.Minute) { + putToSleepRemote(ctx, db, q, svc, apiPort) + } + continue + } containers, err := engine.ListServiceContainers(ctx, svc.ID.String()) if err != nil || len(containers) == 0 { continue @@ -125,6 +139,65 @@ func sweepSleeping(ctx context.Context, db *database.DB, dc *docker.Client, engi } } +// remoteLiveContainers returns inventory rows whose replicas are alive on +// node agents — the signal that idleness must be measured remotely. +func remoteLiveContainers(ctx context.Context, q *sqlcdb.Queries, serviceID string) []sqlcdb.ListServiceContainersRow { + rows, err := q.ListServiceContainers(ctx, serviceID) + if err != nil { + return nil + } + live := rows[:0] + for _, row := range rows { + var st struct { + State string `json:"state"` + } + if err := json.Unmarshal(row.Status.RawMessage, &st); err == nil && + st.State != "" && st.State != "removed" && st.State != "sleeping" { + live = append(live, row) + } + } + return live +} + +// rowNetBytes reads the cumulative rx+tx the agent reported at its last +// heartbeat — 0 for rows written before the counter existed. +func rowNetBytes(row sqlcdb.ListServiceContainersRow) uint64 { + var res struct { + NetBytes uint64 `json:"net_bytes"` + } + _ = json.Unmarshal(row.Resources.RawMessage, &res) + return res.NetBytes +} + +// putToSleepRemote removes the service's remote replicas via their agents, +// marks the service sleeping, and repoints the domain route at the wake +// endpoint so inbound traffic brings it back. +func putToSleepRemote(ctx context.Context, db *database.DB, q *sqlcdb.Queries, svc sleepCandidate, apiPort int) { + runner := newAgentNodeRunner(db) + for _, agentID := range remoteAgentsForService(ctx, db, svc.ID) { + if err := runner.ControlService(ctx, svc.ID.String(), agentID, "remove"); err != nil { + log.Printf("sleep: remote remove on %s failed: %v", agentID, err) + continue + } + _ = q.DeleteServiceContainersOnAgent(ctx, sqlcdb.DeleteServiceContainersOnAgentParams{ + NodeAgentID: agentID, + ServiceID: svc.ID.String(), + }) + } + tracker.mu.Lock() + delete(tracker.activity, svc.ID) + tracker.mu.Unlock() + _, _ = db.Exec(`UPDATE services SET status = 'sleeping', updated_at = $1 WHERE id = $2`, time.Now(), svc.ID) + log.Printf("sleep: %s (%s) remote replicas removed after idle timeout", svc.Name, svc.ID) + + var domain string + _ = db.QueryRow(`SELECT COALESCE(domain,'') FROM services WHERE id = $1`, svc.ID).Scan(&domain) + domains := serviceDomainNames(db, svc.ID, domain) + if len(domains) > 0 { + writeWakeIngress(svc.ID.String(), domains, apiPort) + } +} + // markIdle returns true when the network counter stopped growing past the // timeout. First sighting only sets the baseline — no idle without history. func markIdle(id uuid.UUID, bytes uint64, timeout time.Duration) bool { @@ -346,18 +419,40 @@ func pruneWakeContainers(ctx context.Context, db *database.DB, dc *docker.Client // idempotent and non-destructive, so no token is required; the endpoint // never mutates configuration. func handleServiceWake(c *gin.Context) { + code, body := triggerServiceWake(c) + if code != 0 { + c.JSON(code, body) + } +} + +// handleServiceWakePage is the remote ingress wake target — Traefik routes +// a sleeping remote service's domain here via the file provider. Every hit +// (re)triggers the wake reconcile and returns a reloading page. +func handleServiceWakePage(c *gin.Context) { + triggerServiceWake(c) + c.Header("Content-Type", "text/html; charset=utf-8") + c.String(http.StatusOK, wakePageHTML) +} + +const wakePageHTML = `Waking up` + + `` + + `

Service is waking up

This Containr service was sleeping and is resuming now. The page reloads automatically.

` + +// triggerServiceWake loads the service and reconciles it back to running +// in the background. Returns (0, nil) when a response was already written +// by an embedded handler path, else the JSON status to send. +func triggerServiceWake(c *gin.Context) (int, gin.H) { dbVal, _ := c.Get("db") db := dbVal.(*database.DB) engineVal, _ := c.Get("deployment_engine") engine, _ := engineVal.(*deployment.DeploymentEngine) if engine == nil { - c.JSON(http.StatusServiceUnavailable, gin.H{"error": "deployment engine unavailable"}) - return + return http.StatusServiceUnavailable, gin.H{"error": "deployment engine unavailable"} } id, err := uuid.Parse(c.Param("id")) if err != nil { - c.JSON(http.StatusBadRequest, gin.H{"error": "invalid service id"}) - return + return http.StatusBadRequest, gin.H{"error": "invalid service id"} } var service Service err = db.QueryRow( @@ -385,12 +480,10 @@ func handleServiceWake(c *gin.Context) { &service.CreatedAt, &service.UpdatedAt, ) if err != nil { - c.JSON(http.StatusNotFound, gin.H{"error": "service not found"}) - return + return http.StatusNotFound, gin.H{"error": "service not found"} } if service.Status == "running" || service.Status == "deployed" { - c.JSON(http.StatusOK, gin.H{"status": service.Status}) - return + return http.StatusOK, gin.H{"status": service.Status} } go func() { ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute) @@ -411,7 +504,7 @@ func handleServiceWake(c *gin.Context) { } _, _ = db.Exec(`UPDATE services SET status = $1, updated_at = $2 WHERE id = $3`, status, time.Now(), service.ID) }() - c.JSON(http.StatusAccepted, gin.H{"status": "waking"}) + return http.StatusAccepted, gin.H{"status": "waking"} } // handleServiceSleep puts a service to sleep immediately (owner+ only). @@ -438,7 +531,12 @@ func handleServiceSleep(c *gin.Context) { Name: service.Name, Status: service.Status, } - go putToSleep(context.Background(), db, dc, engine, svc, apiPortFromContext(c)) + q := sqlcdb.New(db.DB) + if len(remoteLiveContainers(c.Request.Context(), q, service.ID.String())) > 0 { + go putToSleepRemote(context.Background(), db, q, svc, apiPortFromContext(c)) + } else { + go putToSleep(context.Background(), db, dc, engine, svc, apiPortFromContext(c)) + } c.JSON(http.StatusAccepted, gin.H{"status": "sleeping"}) } diff --git a/app/backend/internal/database/sqlcdb/agents.sql.go b/app/backend/internal/database/sqlcdb/agents.sql.go index d215260..f553328 100644 --- a/app/backend/internal/database/sqlcdb/agents.sql.go +++ b/app/backend/internal/database/sqlcdb/agents.sql.go @@ -765,8 +765,36 @@ func (q *Queries) ListSchedulableAgentsMatching(ctx context.Context, dollar_1 js return items, nil } +const listServiceAgents = `-- name: ListServiceAgents :many +SELECT DISTINCT node_agent_id FROM container_instances WHERE service_id = $1 +` + +// Distinct agents holding inventory rows for a service. +func (q *Queries) ListServiceAgents(ctx context.Context, serviceID string) ([]string, error) { + rows, err := q.db.QueryContext(ctx, listServiceAgents, serviceID) + if err != nil { + return nil, err + } + defer rows.Close() + items := []string{} + for rows.Next() { + var node_agent_id string + if err := rows.Scan(&node_agent_id); err != nil { + return nil, err + } + items = append(items, node_agent_id) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + const listServiceContainers = `-- name: ListServiceContainers :many -SELECT id, name, node_agent_id, status FROM container_instances +SELECT id, name, node_agent_id, status, ports, resources FROM container_instances WHERE service_id = $1 ` @@ -775,6 +803,8 @@ type ListServiceContainersRow struct { Name string `json:"name"` NodeAgentID string `json:"node_agent_id"` Status pqtype.NullRawMessage `json:"status"` + Ports pqtype.NullRawMessage `json:"ports"` + Resources pqtype.NullRawMessage `json:"resources"` } func (q *Queries) ListServiceContainers(ctx context.Context, serviceID string) ([]ListServiceContainersRow, error) { @@ -791,6 +821,8 @@ func (q *Queries) ListServiceContainers(ctx context.Context, serviceID string) ( &i.Name, &i.NodeAgentID, &i.Status, + &i.Ports, + &i.Resources, ); err != nil { return nil, err } @@ -1005,6 +1037,53 @@ func (q *Queries) UpdateAgentHeartbeat(ctx context.Context, arg UpdateAgentHeart return err } +const updateContainerPorts = `-- name: UpdateContainerPorts :exec +UPDATE container_instances SET status = $2, ports = $3, updated_at = NOW() +WHERE id = $1 +` + +type UpdateContainerPortsParams struct { + ID string `json:"id"` + Status pqtype.NullRawMessage `json:"status"` + Ports pqtype.NullRawMessage `json:"ports"` +} + +// Agent reports the real host bindings after create — persist them with +// the status flip so ingress can route to the assigned port. +func (q *Queries) UpdateContainerPorts(ctx context.Context, arg UpdateContainerPortsParams) error { + _, err := q.db.ExecContext(ctx, updateContainerPorts, arg.ID, arg.Status, arg.Ports) + return err +} + +const updateContainerStateByName = `-- name: UpdateContainerStateByName :exec +UPDATE container_instances SET + status = jsonb_set(status, '{state}', $3::jsonb), + resources = jsonb_set(COALESCE(resources, '{}'::jsonb), '{net_bytes}', $4::jsonb), + updated_at = NOW() +WHERE name = $1 AND node_agent_id = $2 + AND status->>'state' IS DISTINCT FROM 'removed' +` + +type UpdateContainerStateByNameParams struct { + Name string `json:"name"` + NodeAgentID string `json:"node_agent_id"` + Column3 json.RawMessage `json:"column_3"` + Column4 json.RawMessage `json:"column_4"` +} + +// Agent heartbeat reports real docker state + net counters; reconcile +// inventory for rows this node owns. Tombstones win — never resurrect a +// removed replica. $4 is the cumulative rx+tx byte counter. +func (q *Queries) UpdateContainerStateByName(ctx context.Context, arg UpdateContainerStateByNameParams) error { + _, err := q.db.ExecContext(ctx, updateContainerStateByName, + arg.Name, + arg.NodeAgentID, + arg.Column3, + arg.Column4, + ) + return err +} + const updateContainerStatus = `-- name: UpdateContainerStatus :exec UPDATE container_instances SET status = $2, updated_at = NOW() WHERE id = $1 diff --git a/app/backend/internal/database/sqlcdb/querier.go b/app/backend/internal/database/sqlcdb/querier.go index f5b57e7..6d90634 100644 --- a/app/backend/internal/database/sqlcdb/querier.go +++ b/app/backend/internal/database/sqlcdb/querier.go @@ -98,6 +98,8 @@ type Querier interface { ListSchedulableAgents(ctx context.Context) ([]ListSchedulableAgentsRow, error) // Only agents carrying every required placement tag (jsonb array containment). ListSchedulableAgentsMatching(ctx context.Context, dollar_1 json.RawMessage) ([]ListSchedulableAgentsMatchingRow, error) + // Distinct agents holding inventory rows for a service. + ListServiceAgents(ctx context.Context, serviceID string) ([]string, error) ListServiceContainers(ctx context.Context, serviceID string) ([]ListServiceContainersRow, error) ListServiceTemplatesByCategoryForUser(ctx context.Context, arg ListServiceTemplatesByCategoryForUserParams) ([]ServiceTemplate, error) ListServiceTemplatesForUser(ctx context.Context, ownerID uuid.NullUUID) ([]ServiceTemplate, error) @@ -134,6 +136,13 @@ type Querier interface { UpdateAgentHeartbeat(ctx context.Context, arg UpdateAgentHeartbeatParams) error UpdateBackupTargetByIDAndUser(ctx context.Context, arg UpdateBackupTargetByIDAndUserParams) error UpdateBanner(ctx context.Context, arg UpdateBannerParams) (Banner, error) + // Agent reports the real host bindings after create — persist them with + // the status flip so ingress can route to the assigned port. + UpdateContainerPorts(ctx context.Context, arg UpdateContainerPortsParams) error + // Agent heartbeat reports real docker state + net counters; reconcile + // inventory for rows this node owns. Tombstones win — never resurrect a + // removed replica. $4 is the cumulative rx+tx byte counter. + UpdateContainerStateByName(ctx context.Context, arg UpdateContainerStateByNameParams) error UpdateContainerStatus(ctx context.Context, arg UpdateContainerStatusParams) error UpdateDatabaseServiceNameAndPlanByIDAndUser(ctx context.Context, arg UpdateDatabaseServiceNameAndPlanByIDAndUserParams) error UpdateDatabaseServiceNameByIDAndUser(ctx context.Context, arg UpdateDatabaseServiceNameByIDAndUserParams) error diff --git a/app/backend/internal/deployment/engine.go b/app/backend/internal/deployment/engine.go index df501c9..e712bb7 100644 --- a/app/backend/internal/deployment/engine.go +++ b/app/backend/internal/deployment/engine.go @@ -32,6 +32,9 @@ type NodeRunner interface { // RemoteRuntimeState reads the service's remote replicas from inventory // across all agents. nil/empty means nothing runs remotely. RemoteRuntimeState(ctx context.Context, serviceID string) (*RuntimeState, error) + // RetireService removes every remote replica of the service across all + // agents — used when a service returns to local placement. + RetireService(ctx context.Context, serviceID string) error } type DeploymentEngine struct { diff --git a/app/backend/internal/deployment/runtime.go b/app/backend/internal/deployment/runtime.go index ba3168d..dd78d65 100644 --- a/app/backend/internal/deployment/runtime.go +++ b/app/backend/internal/deployment/runtime.go @@ -182,6 +182,12 @@ func (de *DeploymentEngine) ReconcileService(ctx context.Context, spec RuntimeSp } return de.nodeRunner.ReconcileAuto(ctx, spec) } + // Local placement: retire any remote leftovers from a previous pin, + // spread, or tag affinity before reconciling local replicas. + if de.nodeRunner != nil { + _ = de.nodeRunner.RetireService(ctx, spec.ServiceID) + } + if spec.Replicas < 1 { spec.Replicas = 1 } diff --git a/app/backend/sqlc/queries/agents.sql b/app/backend/sqlc/queries/agents.sql index e649893..4358cd3 100644 --- a/app/backend/sqlc/queries/agents.sql +++ b/app/backend/sqlc/queries/agents.sql @@ -110,6 +110,12 @@ RETURNING *; UPDATE container_instances SET status = $2, updated_at = NOW() WHERE id = $1; +-- name: UpdateContainerPorts :exec +-- Agent reports the real host bindings after create — persist them with +-- the status flip so ingress can route to the assigned port. +UPDATE container_instances SET status = $2, ports = $3, updated_at = NOW() +WHERE id = $1; + -- name: GetLastAgentCommandByType :one SELECT * FROM agent_commands WHERE node_agent_id = $1 AND type = $2 @@ -174,7 +180,7 @@ LIMIT 1; UPDATE node_agents SET tags = $2::jsonb, updated_at = NOW() WHERE id = $1; -- name: ListServiceContainers :many -SELECT id, name, node_agent_id, status FROM container_instances +SELECT id, name, node_agent_id, status, ports, resources FROM container_instances WHERE service_id = $1; -- name: SetAgentSchedulable :exec @@ -197,5 +203,20 @@ ON CONFLICT (id) DO UPDATE SET -- name: DeleteServiceContainersOnAgent :exec DELETE FROM container_instances WHERE node_agent_id = $1 AND service_id = $2; +-- name: UpdateContainerStateByName :exec +-- Agent heartbeat reports real docker state + net counters; reconcile +-- inventory for rows this node owns. Tombstones win — never resurrect a +-- removed replica. $4 is the cumulative rx+tx byte counter. +UPDATE container_instances SET + status = jsonb_set(status, '{state}', $3::jsonb), + resources = jsonb_set(COALESCE(resources, '{}'::jsonb), '{net_bytes}', $4::jsonb), + updated_at = NOW() +WHERE name = $1 AND node_agent_id = $2 + AND status->>'state' IS DISTINCT FROM 'removed'; + +-- name: ListServiceAgents :many +-- Distinct agents holding inventory rows for a service. +SELECT DISTINCT node_agent_id FROM container_instances WHERE service_id = $1; + -- name: ScrubCommandPayload :exec UPDATE agent_commands SET payload = $2, updated_at = NOW() WHERE id = $1; diff --git a/app/frontend/src/generated/docs-snapshot.ts b/app/frontend/src/generated/docs-snapshot.ts index c1c9ce6..1307ebf 100644 --- a/app/frontend/src/generated/docs-snapshot.ts +++ b/app/frontend/src/generated/docs-snapshot.ts @@ -40,7 +40,7 @@ export const bundledDocs: BundledDoc[] = [ "path": "guides/CLI.md", "section": "guides", "title": "Containr CLI", - "markdown": "# Containr CLI\n\nThe CLI controls every part of a Containr instance — the same API surface\nthe web UI uses. Build it with `scripts/build-cli.sh` or\n`go build ./cmd/cli` from `app/backend`.\n\n## Authentication\n\nCreate a personal access token in the web UI under\n**Settings → Personal Access Tokens** (or `containr tokens create`), then:\n\n```bash\ncontainr auth login cnp_... # verify + store\ncontainr auth login cnp_... --url https://containr.example.com\ncontainr auth login cnp_... --name production # named profile\ncontainr auth status # show active profile + identity\ncontainr auth logout\n```\n\nResolution order for every command: `--token`/`--api-url` flags >\n`CONTAINR_TOKEN`/`CONTAINR_API_URL` env > active profile >\n`http://localhost:8080/api/v1`. The active profile is `--profile` >\n`CONTAINR_PROFILE` > `current_profile` in `~/.containr.yaml`.\n\n## Output and scripting\n\n- `--json` prints the raw API response — safe to pipe and parse.\n- `--yes` / `-y` skips confirmation on destructive commands (delete,\n revoke, restore, failover). Non-interactive shells refuse destructive\n commands without it.\n- Exit codes are stable: `0` ok, `1` error, `2` auth/permission,\n `3` not found, `4` validation, `5` unreachable.\n\n## Commands\n\n```\nauth login|logout|status token auth and profiles\ntokens list|create|revoke personal access tokens\nprojects list|get|create|delete\nservices list|get|create|update|delete|start|stop|restart|redeploy\nservices clone|move duplicate config or re-parent a service\nservices domains list|add|remove|default|check multi-domain + DNS preflight\nregistries list|add|update|remove private-image pull credentials\nvolumes list|delete docker volume inventory + cleanup (admin)\ndeploy trigger a deployment (--no-cache, --commit-hash, --branch)\ndeployments list|get|logs|rollback|cancel\nlogs [--follow] runtime logs (SSE follow)\nexec -- one-off container command\nvariables list|set|unset env vars (secrets masked; --redeploy applies them)\nvariables shared list|set|unset project shared vars — referenced as ${{shared.KEY}}\ndatabases list|get|create|update|delete|action|backup|restore|download-backup|import-backup\n import-backup uploads an archive as a restore\n point; ships offsite too when a backup target is assigned\ndatabases list|get|create|update|delete|action|backup|restore|download-backup\n create --public exposes the port on all interfaces;\n update --public/--public=false toggles it (container recreates, data persists)\ndatabases register register an external database (--type --host --port --database --username --password --ssl; probed before storing)\ndatabases test-connection probe a connection — by id, or ad-hoc flags\nbackup-targets list|add|update|remove|test\n S3-compatible archive destinations; add/update probe the bucket\n first, empty --access-key/--secret-key on update keep stored keys;\n update --backup-target assigns a target (\"\" clears it)\ncron list|get|create|delete|trigger|executions\ntemplates list|get|create|delete|deploy|plan|import-compose|deploy-graph\n deploy --name --var K=V; plan dry-resolves expressions;\n import-compose converts a compose file into\n a v2 graph config; deploy-graph \n deploys an ad-hoc service graph\nimport compose \n deploys a docker-compose file straight into a project —\n no template row. DB images become managed databases\n attached under their service name; --var K=V overrides\n template variables\nnodes list|get|delete|tokens node agents + onboarding tokens (admin)\nnodes add SSH bootstrap — one session installs the agent, then never needed again (--print for manual runs)\nnodes update|prune|commands rename/auto-prune, bounded docker prune, command history\nmigrate railway pull a Railway project (services, vars, domains) via GraphQL and recreate it — --dry-run prints the plan\nmigrate dflow import a dflow project (services, vars, volumes, domains) via its Payload REST API — needs --url + --api-key, --dry-run prints the plan\nscaling status|policies|services|scale\nha status|alerts|health|policies|enable|disable|failover\nsecurity scan|vulnerabilities|metrics\ngateway services|keys|analytics (admin)\nadmin overview|users|settings|audit-logs\nnotifications list|read\nwebhooks list|create|update|delete|deliveries|test\n outbound signed webhooks — create \n --event 'service.*' (repeatable, '*' = all);\n deliveries are HMAC-signed (X-Containr-Signature)\noperations live ops view: active deploys, queue depth,\n 24h failures, cron runs, backups\nactivity enriched activity feed; --severity/--category/\n --resource/--project/--limit filters\nshell interactive PTY shell inside the service\n container (bash→sh fallback; resizes follow)\nbanners list|create|update|delete announcement banners (admin);\n active banners print on `auth status`\ninvites list|create|revoke team invite links (admin); create\n prints the one-time acceptance URL\nadmin impersonate mint a 15-min token acting as a user\nadmin github-app status|manifest|convert\n self-provision the instance GitHub App\nup deploy the current directory\n```\n\n## containr up\n\nRun inside a git checkout: uses `remote.origin.url` + the current branch\nas the build source, creates a service named after the directory when none\nexists, then triggers a deployment.\n\n```bash\ncontainr up --project \ncontainr up --project --image ghcr.io/example/app:latest # image, no build\n```\n\nSet `project_id` on the active profile to omit `--project` (stored in\n`~/.containr.yaml` under `profiles..project_id`).\n\n## Service volumes\n\n`services update` manages mounts; flags replace the whole mount list:\n\n```bash\ncontainr services update --volume data:/data --bind /srv/cfg:/etc/app:ro\ncontainr services update --clear-volumes\n```\n\n`type` is `volume` (docker named volume, auto-created) or `bind` (host\npath). `target` must be absolute. Mounts apply on the next deploy or\nredeploy — `containr deploy ` to apply immediately.\n\n`volumes list` shows the node's docker volume inventory with in-use\nflags; `volumes delete ` removes an unused volume (admin scope).\n\n## Node placement\n\nPin a service to a registered node agent, or let the scheduler pick the\nleast-loaded online node:\n\n```bash\ncontainr services create --project --name api --image ghcr.io/x/api:v1 --node \ncontainr services update --node auto # least-loaded online node\ncontainr services update --node local # move back to this server\n```\n\n`auto` scores nodes by real headroom — lowest memory utilisation, then\ncpu, then container count. Or spread replicas across every online,\nschedulable node (mutually exclusive with a pin):\n\n```bash\ncontainr services update --replicas 4 --spread # replica i → node[i % n]\ncontainr services update --spread=false # back to single placement\n```\n\nConstrain `auto`/`spread` placement to nodes carrying operator tags —\n`nodes update` labels a node, `--placement-tags` requires every listed tag\n(mutually exclusive with a pin; an empty value clears the requirement):\n\n```bash\ncontainr nodes update --tags gpu,eu-west\ncontainr services update --placement-tags gpu # only gpu nodes are eligible\n```\n\nRemote nodes run registry-pulled images only — git builds, domains\n(Traefik lives on the host), and sleep detection stay local. Start,\nstop, restart, redeploy, and delete all fan out to the pinned node's\nagent. `services get` shows the resolved `node_name`.\n\nTake a node out of rotation or empty it entirely (admin):\n\n```bash\ncontainr nodes cordon # no new pins; running replicas stay\ncontainr nodes uncordon \ncontainr nodes drain # cordon + remove its containers + unpin services\ncontainr nodes upgrade # agent pulls its replacement from the server, re-execs\n```\n\n`nodes upgrade` swaps the agent binary for the one bundled in the running\nserver image — the agent downloads it over the API, verifies the sha256 the\nserver computed, sanity-checks it, atomically replaces its own executable,\nand re-execs. The node must be online and the server must ship a binary for\nits architecture (`linux-amd64`/`linux-arm64`).\n\n## Service domains and access\n\nA service can carry multiple hostnames; the default is mirrored to\n`services.domain`:\n\n```bash\ncontainr services domains add app.example.com --default\ncontainr services domains check # DNS preflight per domain\ncontainr services domains remove \n```\n\nDNS check compares each domain to the expected target — set\n`app_settings.public_ip` (or `PUBLIC_IP` env) so `wrong-target` is\nmeaningful; without it any resolvable domain reports `ok`.\n\nAccess gates apply via Traefik middlewares (no extra proxy):\n\n```bash\ncontainr services update --maintenance on\ncontainr services update --basic-auth admin:secret --basic-auth ops:pw2\n```\n\nMaintenance mode redirects all traffic to `/api/v1/maintenance`\n— set `app_settings.base_url` or `BASE_URL`; without it the service is\ntaken offline with empty responses. Basic-auth credentials are stored\nbcrypt-hashed (htpasswd) and gate every request.\n\n## Build strategy and resources\n\nSource-built services pick a builder per service:\n\n```bash\ncontainr services update --builder railpack # auto|railpack|nixpacks|dockerfile|static\ncontainr services update --builder static --static-cmd \"npm run build\" --static-dir dist\n```\n\n`auto` detects from repo contents (Dockerfile → railpack → nixpacks).\n`static` builds assets in a node stage and serves them from nginx —\n`--static-cmd` runs inside the build stage (default\n`npm ci && npm run build`), `--static-dir` is copied into the nginx image\n(default `dist`, exposed on port 80).\n\nResource reservations complement the hard `cpu`/`memory` limits:\n\n```bash\ncontainr services update --cpu-reserve 0.25 --memory-reserve 128Mi\n```\n\n`cpu`/`memory` are hard caps (NanoCPUs/Memory); `cpu_reserve`/\n`memory_reserve` are soft hints (CPU shares, MemoryReservation). Empty\nstring clears. `app_settings.default_cpu`/`default_memory` change the\ninstance-level defaults used at service creation;\n`app_settings.capacity_policy=block` fails deployments whose requested\nmemory exceeds node capacity (default `warn` logs only).\n\n## Sleep mode (scale-to-zero)\n\nServices can scale to zero after an idle window and resume on traffic:\n\n```bash\ncontainr services update --sleep --sleep-idle 15 # enable, 15 min idle timeout\ncontainr services update --sleep=false # disable\ncontainr services sleep # sleep now\ncontainr services wake # resume now\ncontainr services env-check # flag empty vars + unresolved ${{refs}}\n```\n\nIdleness is measured as flat container network I/O over the configured\nwindow (swept every 30s). Sleeping removes the workload containers and —\nwhen the service has domains — installs a busybox placeholder that holds\nthe Traefik route at higher priority, serves a reloading \"waking\" page,\nand pings `GET /api/v1/internal/wake/:id` to reconcile the service back.\nThe API must be reachable from containers for the callback (containerized\ndeploys work via the shared network; bare-metal needs `HOST=0.0.0.0`).\n\n## Clone, move, registries\n\n`services clone` duplicates config, volumes, domains, access gates,\nbuilder settings, and variables — ciphertext secret values are copied\nverbatim (no decrypt/re-encrypt). The clone starts stopped:\n\n```bash\ncontainr services clone --name staging-api --project --environment preview\ncontainr services move # name conflicts → 409\n```\n\nNote: cloned domains are verbatim copies — the same hostname attached to\ntwo services makes Traefik routing ambiguous. Re-domain or undeploy one.\n\n`registries` stores private-image pull credentials, owner-scoped and\nencrypted at rest. Matched by image host on every image-sourced deploy:\n\n```bash\ncontainr registries add ghcr ghcr.io --username --password \ncontainr registries add hub docker.io --username --password # Docker Hub\ncontainr registries update --username # empty --password keeps stored\ncontainr registries remove \n```\n\n## MCP server\n\n`containr-mcp` (`go build ./cmd/mcp`) exposes the same API as MCP tools\nover stdio — every command group above maps to a `containr_*` tool\n(~60 tools). Destructive tools require the caller to pass\n`confirm: true`; results are pretty JSON with `isError` on failures.\n\nClient configuration (Claude Code, Cursor, Devin, any stdio MCP host):\n\n```json\n{\n \"mcpServers\": {\n \"containr\": {\n \"command\": \"containr-mcp\",\n \"env\": {\n \"CONTAINR_API_URL\": \"https://containr.example.com\",\n \"CONTAINR_TOKEN\": \"cnp_...\"\n }\n }\n }\n}\n```\n\nScope the token deliberately: `read` is enough for inspection agents,\n`write` for deploy automation, `admin` only for platform operations.\n\nDeployments are serialized per service: a second `deploy` on a busy\nservice lands in `queued` state and runs when the active one finishes.\n`deployments cancel ` drops a queued deployment or aborts a running\none (builds/reconciles stop via context cancellation).\n" + "markdown": "# Containr CLI\n\nThe CLI controls every part of a Containr instance — the same API surface\nthe web UI uses. Build it with `scripts/build-cli.sh` or\n`go build ./cmd/cli` from `app/backend`.\n\n## Authentication\n\nCreate a personal access token in the web UI under\n**Settings → Personal Access Tokens** (or `containr tokens create`), then:\n\n```bash\ncontainr auth login cnp_... # verify + store\ncontainr auth login cnp_... --url https://containr.example.com\ncontainr auth login cnp_... --name production # named profile\ncontainr auth status # show active profile + identity\ncontainr auth logout\n```\n\nResolution order for every command: `--token`/`--api-url` flags >\n`CONTAINR_TOKEN`/`CONTAINR_API_URL` env > active profile >\n`http://localhost:8080/api/v1`. The active profile is `--profile` >\n`CONTAINR_PROFILE` > `current_profile` in `~/.containr.yaml`.\n\n## Output and scripting\n\n- `--json` prints the raw API response — safe to pipe and parse.\n- `--yes` / `-y` skips confirmation on destructive commands (delete,\n revoke, restore, failover). Non-interactive shells refuse destructive\n commands without it.\n- Exit codes are stable: `0` ok, `1` error, `2` auth/permission,\n `3` not found, `4` validation, `5` unreachable.\n\n## Commands\n\n```\nauth login|logout|status token auth and profiles\ntokens list|create|revoke personal access tokens\nprojects list|get|create|delete\nservices list|get|create|update|delete|start|stop|restart|redeploy\nservices clone|move duplicate config or re-parent a service\nservices domains list|add|remove|default|check multi-domain + DNS preflight\nregistries list|add|update|remove private-image pull credentials\nvolumes list|delete docker volume inventory + cleanup (admin)\ndeploy trigger a deployment (--no-cache, --commit-hash, --branch)\ndeployments list|get|logs|rollback|cancel\nlogs [--follow] runtime logs (SSE follow)\nexec -- one-off container command\nvariables list|set|unset env vars (secrets masked; --redeploy applies them)\nvariables shared list|set|unset project shared vars — referenced as ${{shared.KEY}}\ndatabases list|get|create|update|delete|action|backup|restore|download-backup|import-backup\n import-backup uploads an archive as a restore\n point; ships offsite too when a backup target is assigned\ndatabases list|get|create|update|delete|action|backup|restore|download-backup\n create --public exposes the port on all interfaces;\n update --public/--public=false toggles it (container recreates, data persists)\ndatabases register register an external database (--type --host --port --database --username --password --ssl; probed before storing)\ndatabases test-connection probe a connection — by id, or ad-hoc flags\nbackup-targets list|add|update|remove|test\n S3-compatible archive destinations; add/update probe the bucket\n first, empty --access-key/--secret-key on update keep stored keys;\n update --backup-target assigns a target (\"\" clears it)\ncron list|get|create|delete|trigger|executions\ntemplates list|get|create|delete|deploy|plan|import-compose|deploy-graph\n deploy --name --var K=V; plan dry-resolves expressions;\n import-compose converts a compose file into\n a v2 graph config; deploy-graph \n deploys an ad-hoc service graph\nimport compose \n deploys a docker-compose file straight into a project —\n no template row. DB images become managed databases\n attached under their service name; --var K=V overrides\n template variables\nnodes list|get|delete|tokens node agents + onboarding tokens (admin)\nnodes add SSH bootstrap — one session installs the agent, then never needed again (--print for manual runs)\nnodes update|prune|commands rename/auto-prune, bounded docker prune, command history\nmigrate railway pull a Railway project (services, vars, domains) via GraphQL and recreate it — --dry-run prints the plan\nmigrate dflow import a dflow project (services, vars, volumes, domains) via its Payload REST API — needs --url + --api-key, --dry-run prints the plan\nscaling status|policies|services|scale\nha status|alerts|health|policies|enable|disable|failover\nsecurity scan|vulnerabilities|metrics\ngateway services|keys|analytics (admin)\nadmin overview|users|settings|audit-logs\nnotifications list|read\nwebhooks list|create|update|delete|deliveries|test\n outbound signed webhooks — create \n --event 'service.*' (repeatable, '*' = all);\n deliveries are HMAC-signed (X-Containr-Signature)\noperations live ops view: active deploys, queue depth,\n 24h failures, cron runs, backups\nactivity enriched activity feed; --severity/--category/\n --resource/--project/--limit filters\nshell interactive PTY shell inside the service\n container (bash→sh fallback; resizes follow)\nbanners list|create|update|delete announcement banners (admin);\n active banners print on `auth status`\ninvites list|create|revoke team invite links (admin); create\n prints the one-time acceptance URL\nadmin impersonate mint a 15-min token acting as a user\nadmin github-app status|manifest|convert\n self-provision the instance GitHub App\nup deploy the current directory\n```\n\n## containr up\n\nRun inside a git checkout: uses `remote.origin.url` + the current branch\nas the build source, creates a service named after the directory when none\nexists, then triggers a deployment.\n\n```bash\ncontainr up --project \ncontainr up --project --image ghcr.io/example/app:latest # image, no build\n```\n\nSet `project_id` on the active profile to omit `--project` (stored in\n`~/.containr.yaml` under `profiles..project_id`).\n\n## Service volumes\n\n`services update` manages mounts; flags replace the whole mount list:\n\n```bash\ncontainr services update --volume data:/data --bind /srv/cfg:/etc/app:ro\ncontainr services update --clear-volumes\n```\n\n`type` is `volume` (docker named volume, auto-created) or `bind` (host\npath). `target` must be absolute. Mounts apply on the next deploy or\nredeploy — `containr deploy ` to apply immediately.\n\n`volumes list` shows the node's docker volume inventory with in-use\nflags; `volumes delete ` removes an unused volume (admin scope).\n\n## Node placement\n\nPin a service to a registered node agent, or let the scheduler pick the\nleast-loaded online node:\n\n```bash\ncontainr services create --project --name api --image ghcr.io/x/api:v1 --node \ncontainr services update --node auto # least-loaded online node\ncontainr services update --node local # move back to this server\n```\n\n`auto` scores nodes by real headroom — lowest memory utilisation, then\ncpu, then container count. Or spread replicas across every online,\nschedulable node (mutually exclusive with a pin):\n\n```bash\ncontainr services update --replicas 4 --spread # replica i → node[i % n]\ncontainr services update --spread=false # back to single placement\n```\n\nConstrain `auto`/`spread` placement to nodes carrying operator tags —\n`nodes update` labels a node, `--placement-tags` requires every listed tag\n(mutually exclusive with a pin; an empty value clears the requirement):\n\n```bash\ncontainr nodes update --tags gpu,eu-west\ncontainr services update --placement-tags gpu # only gpu nodes are eligible\n```\n\nRemote nodes run registry-pulled images only — git builds stay local.\nDomains work remotely: the agent publishes the service port and the\nbackend writes a Traefik file-provider route to the node's mesh IP\n(TRAEFIK_DYNAMIC_DIR — wired by the infra compose). Sleep mode works\ntoo: heartbeats carry per-container net counters, idle remote services\nlose their replicas via the agent, and the route repoints at the\nbackend's wake page — the next hit reconciles the service back. Start,\nstop, restart, redeploy, and delete all fan out to the pinned node's\nagent. `services get` shows the resolved `node_name`.\n\nTake a node out of rotation or empty it entirely (admin):\n\n```bash\ncontainr nodes cordon # no new pins; running replicas stay\ncontainr nodes uncordon \ncontainr nodes drain # cordon + remove its containers + unpin services\ncontainr nodes upgrade # agent pulls its replacement from the server, re-execs\n```\n\n`nodes upgrade` swaps the agent binary for the one bundled in the running\nserver image — the agent downloads it over the API, verifies the sha256 the\nserver computed, sanity-checks it, atomically replaces its own executable,\nand re-execs. The node must be online and the server must ship a binary for\nits architecture (`linux-amd64`/`linux-arm64`).\n\n## Service domains and access\n\nA service can carry multiple hostnames; the default is mirrored to\n`services.domain`:\n\n```bash\ncontainr services domains add app.example.com --default\ncontainr services domains check # DNS preflight per domain\ncontainr services domains remove \n```\n\nDNS check compares each domain to the expected target — set\n`app_settings.public_ip` (or `PUBLIC_IP` env) so `wrong-target` is\nmeaningful; without it any resolvable domain reports `ok`.\n\nAccess gates apply via Traefik middlewares (no extra proxy):\n\n```bash\ncontainr services update --maintenance on\ncontainr services update --basic-auth admin:secret --basic-auth ops:pw2\n```\n\nMaintenance mode redirects all traffic to `/api/v1/maintenance`\n— set `app_settings.base_url` or `BASE_URL`; without it the service is\ntaken offline with empty responses. Basic-auth credentials are stored\nbcrypt-hashed (htpasswd) and gate every request.\n\n## Build strategy and resources\n\nSource-built services pick a builder per service:\n\n```bash\ncontainr services update --builder railpack # auto|railpack|nixpacks|dockerfile|static\ncontainr services update --builder static --static-cmd \"npm run build\" --static-dir dist\n```\n\n`auto` detects from repo contents (Dockerfile → railpack → nixpacks).\n`static` builds assets in a node stage and serves them from nginx —\n`--static-cmd` runs inside the build stage (default\n`npm ci && npm run build`), `--static-dir` is copied into the nginx image\n(default `dist`, exposed on port 80).\n\nResource reservations complement the hard `cpu`/`memory` limits:\n\n```bash\ncontainr services update --cpu-reserve 0.25 --memory-reserve 128Mi\n```\n\n`cpu`/`memory` are hard caps (NanoCPUs/Memory); `cpu_reserve`/\n`memory_reserve` are soft hints (CPU shares, MemoryReservation). Empty\nstring clears. `app_settings.default_cpu`/`default_memory` change the\ninstance-level defaults used at service creation;\n`app_settings.capacity_policy=block` fails deployments whose requested\nmemory exceeds node capacity (default `warn` logs only).\n\n## Sleep mode (scale-to-zero)\n\nServices can scale to zero after an idle window and resume on traffic:\n\n```bash\ncontainr services update --sleep --sleep-idle 15 # enable, 15 min idle timeout\ncontainr services update --sleep=false # disable\ncontainr services sleep # sleep now\ncontainr services wake # resume now\ncontainr services env-check # flag empty vars + unresolved ${{refs}}\n```\n\nIdleness is measured as flat container network I/O over the configured\nwindow (swept every 30s). Sleeping removes the workload containers and —\nwhen the service has domains — installs a busybox placeholder that holds\nthe Traefik route at higher priority, serves a reloading \"waking\" page,\nand pings `GET /api/v1/internal/wake/:id` to reconcile the service back.\nThe API must be reachable from containers for the callback (containerized\ndeploys work via the shared network; bare-metal needs `HOST=0.0.0.0`).\n\n## Clone, move, registries\n\n`services clone` duplicates config, volumes, domains, access gates,\nbuilder settings, and variables — ciphertext secret values are copied\nverbatim (no decrypt/re-encrypt). The clone starts stopped:\n\n```bash\ncontainr services clone --name staging-api --project --environment preview\ncontainr services move # name conflicts → 409\n```\n\nNote: cloned domains are verbatim copies — the same hostname attached to\ntwo services makes Traefik routing ambiguous. Re-domain or undeploy one.\n\n`registries` stores private-image pull credentials, owner-scoped and\nencrypted at rest. Matched by image host on every image-sourced deploy:\n\n```bash\ncontainr registries add ghcr ghcr.io --username --password \ncontainr registries add hub docker.io --username --password # Docker Hub\ncontainr registries update --username # empty --password keeps stored\ncontainr registries remove \n```\n\n## MCP server\n\n`containr-mcp` (`go build ./cmd/mcp`) exposes the same API as MCP tools\nover stdio — every command group above maps to a `containr_*` tool\n(~60 tools). Destructive tools require the caller to pass\n`confirm: true`; results are pretty JSON with `isError` on failures.\n\nClient configuration (Claude Code, Cursor, Devin, any stdio MCP host):\n\n```json\n{\n \"mcpServers\": {\n \"containr\": {\n \"command\": \"containr-mcp\",\n \"env\": {\n \"CONTAINR_API_URL\": \"https://containr.example.com\",\n \"CONTAINR_TOKEN\": \"cnp_...\"\n }\n }\n }\n}\n```\n\nScope the token deliberately: `read` is enough for inspection agents,\n`write` for deploy automation, `admin` only for platform operations.\n\nDeployments are serialized per service: a second `deploy` on a busy\nservice lands in `queued` state and runs when the active one finishes.\n`deployments cancel ` drops a queued deployment or aborts a running\none (builds/reconciles stop via context cancellation).\n" }, { "path": "guides/CLOUDFLARE_SETUP.md", diff --git a/docs/guides/CLI.md b/docs/guides/CLI.md index e9223ea..8aa7282 100644 --- a/docs/guides/CLI.md +++ b/docs/guides/CLI.md @@ -161,8 +161,13 @@ containr nodes update --tags gpu,eu-west containr services update --placement-tags gpu # only gpu nodes are eligible ``` -Remote nodes run registry-pulled images only — git builds, domains -(Traefik lives on the host), and sleep detection stay local. Start, +Remote nodes run registry-pulled images only — git builds stay local. +Domains work remotely: the agent publishes the service port and the +backend writes a Traefik file-provider route to the node's mesh IP +(TRAEFIK_DYNAMIC_DIR — wired by the infra compose). Sleep mode works +too: heartbeats carry per-container net counters, idle remote services +lose their replicas via the agent, and the route repoints at the +backend's wake page — the next hit reconciles the service back. Start, stop, restart, redeploy, and delete all fan out to the pinned node's agent. `services get` shows the resolved `node_name`. diff --git a/infra/docker-compose.yml b/infra/docker-compose.yml index 6f2ac62..7ce46c7 100644 --- a/infra/docker-compose.yml +++ b/infra/docker-compose.yml @@ -7,7 +7,8 @@ services: - "--api.insecure=${TRAEFIK_API_INSECURE:-true}" - "--providers.docker=true" - "--providers.docker.exposedbydefault=false" - - "--providers.file.filename=/etc/traefik/traefik-dynamic.yml" + - "--providers.file.directory=/traefik-dynamic" + - "--providers.file.watch=true" - "--entrypoints.web.address=:80" - "--entrypoints.websecure.address=:443" - "--entrypoints.traefik.address=:8080" @@ -26,7 +27,8 @@ services: volumes: - /var/run/docker.sock:/var/run/docker.sock:ro - ../data/letsencrypt:/letsencrypt - - ./traefik-dynamic.yml:/etc/traefik/traefik-dynamic.yml:ro + - traefik_dynamic:/traefik-dynamic + - ./traefik-dynamic.yml:/traefik-dynamic/00-base.yml:ro networks: - containr-network restart: unless-stopped @@ -100,6 +102,8 @@ services: - JWT_SECRET=${JWT_SECRET:-dev_jwt_secret_key_change_in_production} - CONTAINR_AGENT_AUTH_TOKEN=${CONTAINR_AGENT_AUTH_TOKEN:-} - CONTAINR_AGENT_AUTH_TOKENS=${CONTAINR_AGENT_AUTH_TOKENS:-} + - TRAEFIK_DYNAMIC_DIR=/traefik-dynamic + - TRAEFIK_BACKEND_URL=http://containr-backend:8080 - COOKIE_SECURE=${COOKIE_SECURE:-true} - TRUSTED_PROXY_CIDR=${TRUSTED_PROXY_CIDR:-172.20.0.0/16} - CORS_ORIGINS=${CORS_ORIGINS:-http://localhost,http://localhost:3000} @@ -136,6 +140,7 @@ services: - GITEA_BASE_URL=${GITEA_BASE_URL:-https://gitea.example.com} volumes: - /var/run/docker.sock:/var/run/docker.sock + - traefik_dynamic:/traefik-dynamic group_add: - "${DOCKER_GID:-0}" networks: @@ -213,6 +218,8 @@ volumes: driver: local dragonfly_data: driver: local + traefik_dynamic: + driver: local networks: containr-network: