From 9de586f1774e0328b14a7739c5ff099465547f25 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Dvo=C5=99=C3=A1k?= <150935816+Dvorinka@users.noreply.github.com> Date: Fri, 9 Oct 2026 17:15:58 +0200 Subject: [PATCH] feat(nodes): agent self-upgrade (#57) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit POST /agents/:id/upgrade enqueues a self_upgrade command carrying the target platform, the sha256 the server computed over its bundled binary, and the server version. The agent downloads the replacement from the existing /api/agents/download/:platform endpoint, verifies the checksum, sanity-checks the new binary (--version), atomically swaps its own executable, reports the command result, then re-execs via syscall.Exec β€” no supervisor or restart plumbing required. Agents stamp main.version via ldflags at image build, expose --version, and now send version at register so node_agents.version refreshes on re-register (older agents without the field keep their stored value). The server rejects upgrades for offline nodes and for platforms with no bundled binary (linux-amd64/linux-arm64 ship in the image). Surfaces: nodes upgrade CLI, containr_nodes_upgrade MCP (confirm-gated), node-detail page button, OpenAPI. Verified live: v0.0.3 agent upgraded to v0.0.4 in place, re-registered with the new version; offline node returns 409 NODE_OFFLINE. Generated with [Devin](https://devin.ai) Co-authored-by: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- ROADMAP2.md | 12 +++ app/backend/Dockerfile | 4 +- app/backend/cmd/agent/main.go | 90 ++++++++++++++++++- app/backend/internal/api/agents.go | 90 +++++++++++++++++-- app/backend/internal/cli/commands/nodes.go | 20 ++++- app/backend/internal/mcpserver/server.go | 1 + .../src/features/nodes/NodeDetailPage.tsx | 15 ++++ app/frontend/src/generated/api-types.ts | 64 +++++++++++++ app/frontend/src/generated/docs-snapshot.ts | 2 +- app/frontend/src/lib/api-client.ts | 6 ++ docs/api/openapi.yaml | 40 +++++++++ docs/guides/CLI.md | 7 ++ 12 files changed, 337 insertions(+), 14 deletions(-) diff --git a/ROADMAP2.md b/ROADMAP2.md index 01df32c..6fdd6f0 100644 --- a/ROADMAP2.md +++ b/ROADMAP2.md @@ -301,6 +301,18 @@ and wired refs, one click or one CLI call. documented ceilings. Remaining: `scheduling_rules` affinity policies, deployment status reported back from agent results, spread weighting by real capacity. +- [x] **Agent self-upgrade** β€” `POST /agents/:id/upgrade` enqueues a + `self_upgrade` command carrying platform + sha256 + server version; + the agent downloads its replacement from `GET /agents/download/ + :platform`, verifies the checksum, sanity-checks `--version`, + atomically swaps its own executable, reports the result, then + re-execs (`syscall.Exec`) β€” no supervisor needed. Agents now stamp + `main.version` via ldflags at image build and report it at + register/heartbeat (`node_agents.version` refreshes on re-register; + older agents keep their stored value). Surfaces: `nodes upgrade` + CLI, `containr_nodes_upgrade` MCP (confirm-gated), node-detail page + button (online-gated). Ceilings: linux amd64/arm64 only; agents on + other platforms get a `NO_BINARY` 409. - [🟑] **Build-on-node** β€” private registry pulls shipped: the dispatch payload carries the project owner's `registries` credentials for the image host; the agent `docker login`s for the pull and logs out after, diff --git a/app/backend/Dockerfile b/app/backend/Dockerfile index 2c28db1..6bbb1cd 100644 --- a/app/backend/Dockerfile +++ b/app/backend/Dockerfile @@ -14,8 +14,8 @@ RUN CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo \ -ldflags "-X containr/internal/api.Version=${VERSION}" -o main cmd/server/main.go # Node-agent binaries served by the API for SSH/bootstrap installs. -RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o /agents/containr-agent-linux-amd64 ./cmd/agent && \ - CGO_ENABLED=0 GOOS=linux GOARCH=arm64 go build -o /agents/containr-agent-linux-arm64 ./cmd/agent +RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -ldflags "-X main.version=${VERSION}" -o /agents/containr-agent-linux-amd64 ./cmd/agent && \ + CGO_ENABLED=0 GOOS=linux GOARCH=arm64 go build -ldflags "-X main.version=${VERSION}" -o /agents/containr-agent-linux-arm64 ./cmd/agent # Build embedded Better Auth runtime dependencies FROM node:22-alpine AS auth-builder diff --git a/app/backend/cmd/agent/main.go b/app/backend/cmd/agent/main.go index 9fc79a8..48d2124 100644 --- a/app/backend/cmd/agent/main.go +++ b/app/backend/cmd/agent/main.go @@ -3,14 +3,18 @@ package main import ( "bytes" "context" + "crypto/sha256" + "encoding/hex" "encoding/json" "errors" "fmt" + "io" "log" "net" "net/http" "os" "os/exec" + "path/filepath" "runtime" "strconv" "strings" @@ -104,7 +108,15 @@ type registerResponse struct { Status string `json:"status"` } +// version is stamped at build time via -ldflags "-X main.version=…". +// The server reports it on upgrades so agents can show what they run. +var version = "dev" + func main() { + if len(os.Args) > 1 && (os.Args[1] == "--version" || os.Args[1] == "version") { + fmt.Printf("containr-agent %s\n", version) + return + } cfg, err := loadConfig() if err != nil { log.Fatal(err) @@ -167,6 +179,7 @@ func register(ctx context.Context, client *http.Client, cfg agentConfig) (string "ip_address": cfg.IP, "port": cfg.Port, "capabilities": detectCapabilities(), + "version": firstEnv("CONTAINR_AGENT_VERSION", version), // Always send mesh so a removed interface clears stored state. "mesh": detectMeshInterfaces(), } @@ -190,7 +203,7 @@ func sendHeartbeat(ctx context.Context, client *http.Client, cfg agentConfig, ag "container_count": dockerContainerCount(), "system_load": readSystemLoad(), "uptime": readSystemUptime(), - "version": firstEnv("CONTAINR_AGENT_VERSION", "dev"), + "version": firstEnv("CONTAINR_AGENT_VERSION", version), } return postJSON(ctx, client, cfg, "/api/agents/heartbeat", payload, nil) } @@ -216,7 +229,13 @@ func processCommands(ctx context.Context, client *http.Client, cfg agentConfig, return err } for _, item := range list.Commands { - result, runErr := executeCommand(item) + var result, reexecPath string + var runErr error + if item.Type == "self_upgrade" { + result, reexecPath, runErr = selfUpgrade(cfg, item.Payload) + } else { + result, runErr = executeCommand(item) + } status := "completed" errText := "" if runErr != nil { @@ -232,6 +251,12 @@ func processCommands(ctx context.Context, client *http.Client, cfg agentConfig, if err := postJSON(ctx, client, cfg, path, payload, nil); err != nil { log.Printf("report command result failed: command=%s err=%v", item.ID, err) } + // Re-exec AFTER the result posts β€” the new binary picks up the loop. + if reexecPath != "" { + if err := syscall.Exec(reexecPath, os.Args, os.Environ()); err != nil { + log.Fatalf("re-exec after upgrade failed: %v", err) + } + } } return nil } @@ -257,6 +282,67 @@ func executeCommand(item command) (string, error) { } } +// selfUpgrade downloads the agent binary the server bundles for this +// platform, verifies it, swaps it in atomically, and returns the path to +// re-exec. On any failure the running binary is left untouched. +func selfUpgrade(cfg agentConfig, payload map[string]interface{}) (string, string, error) { + platform := firstString(stringValue(payload["platform"]), runtime.GOOS+"-"+runtime.GOARCH) + + exe, err := os.Executable() + if err != nil { + return "", "", fmt.Errorf("resolve own path: %w", err) + } + if resolved, err := filepath.EvalSymlinks(exe); err == nil { + exe = resolved + } + + url := fmt.Sprintf("%s/api/agents/download/%s", cfg.APIURL, platform) + req, err := http.NewRequest(http.MethodGet, url, nil) + if err != nil { + return "", "", err + } + resp, err := (&http.Client{Timeout: 2 * time.Minute}).Do(req) + if err != nil { + return "", "", fmt.Errorf("download %s: %w", platform, err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return "", "", fmt.Errorf("download %s: server returned %d", platform, resp.StatusCode) + } + + newPath := exe + ".new" + out, err := os.OpenFile(newPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o755) + if err != nil { + return "", "", fmt.Errorf("write %s: %w", newPath, err) + } + h := sha256.New() + if _, err := io.Copy(io.MultiWriter(out, h), resp.Body); err != nil { + out.Close() + os.Remove(newPath) + return "", "", fmt.Errorf("download write: %w", err) + } + out.Close() + + if want := stringValue(payload["sha256"]); want != "" { + if got := hex.EncodeToString(h.Sum(nil)); got != want { + os.Remove(newPath) + return "", "", fmt.Errorf("sha256 mismatch: got %s want %s", got, want) + } + } + + // Sanity: the new binary must at least report its version. + if verOut, err := exec.Command(newPath, "--version").CombinedOutput(); err != nil { + os.Remove(newPath) + return "", "", fmt.Errorf("new binary failed sanity check: %s", strings.TrimSpace(string(verOut))) + } + + if err := os.Rename(newPath, exe); err != nil { + os.Remove(newPath) + return "", "", fmt.Errorf("swap binary: %w", err) + } + return fmt.Sprintf("upgraded to %s (%s)", firstString(stringValue(payload["version"]), "latest"), platform), exe, nil +} + // dockerPrune runs a bounded docker system prune. Volumes are excluded // unless the payload explicitly opts in β€” a stray volume prune destroys // database data. diff --git a/app/backend/internal/api/agents.go b/app/backend/internal/api/agents.go index b3bfdfb..554cfd9 100644 --- a/app/backend/internal/api/agents.go +++ b/app/backend/internal/api/agents.go @@ -11,6 +11,7 @@ import ( "encoding/json" "errors" "fmt" + "io" "log" "math" "net/http" @@ -353,6 +354,7 @@ func (h *NodeAgentHandler) RegisterAgent(c *gin.Context) { IPAddress string `json:"ip_address" binding:"required"` Port int `json:"port" binding:"required"` Capabilities AgentCapabilities `json:"capabilities" binding:"required"` + Version string `json:"version"` AuthToken string `json:"auth_token"` Mesh map[string]string `json:"mesh"` } @@ -392,13 +394,14 @@ func (h *NodeAgentHandler) RegisterAgent(c *gin.Context) { } } updated, err := h.q.UpdateAgent(ctx, sqlcdb.UpdateAgentParams{ - ID: existing.ID, - Name: req.Name, - Hostname: existing.Hostname, - IpAddress: existing.IpAddress, - Port: int32(req.Port), - Status: sql.NullString{String: "connecting", Valid: true}, - Version: existing.Version, + ID: existing.ID, + Name: req.Name, + Hostname: existing.Hostname, + IpAddress: existing.IpAddress, + Port: int32(req.Port), + Status: sql.NullString{String: "connecting", Valid: true}, + // Older agents don't report a version β€” keep whatever's stored. + Version: sql.NullString{String: firstNonEmpty(req.Version, existing.Version.String), Valid: true}, Capabilities: rawJSON(req.Capabilities), Resources: existing.Resources, LastHeartbeat: sql.NullTime{Time: time.Now(), Valid: true}, @@ -450,7 +453,7 @@ func (h *NodeAgentHandler) RegisterAgent(c *gin.Context) { IpAddress: req.IPAddress, Port: int32(req.Port), Status: sql.NullString{String: "connecting", Valid: true}, - Version: sql.NullString{}, + Version: sql.NullString{String: req.Version, Valid: req.Version != ""}, Capabilities: rawJSON(req.Capabilities), Resources: rawJSON(resources), LastHeartbeat: sql.NullTime{Time: time.Now(), Valid: true}, @@ -911,6 +914,76 @@ func (h *NodeAgentHandler) PruneAgent(c *gin.Context) { c.JSON(http.StatusAccepted, gin.H{"command": command}) } +// UpgradeAgent enqueues a self_upgrade command: the agent downloads its +// replacement binary from this server's bundled /download endpoint, +// verifies the sha256, swaps it in, and re-execs. Fails fast when no +// bundled binary exists for the agent's architecture. +func (h *NodeAgentHandler) UpgradeAgent(c *gin.Context) { + ctx := c.Request.Context() + id := c.Param("id") + agent, err := h.q.GetAgent(ctx, id) + if err != nil { + respondError(c, http.StatusNotFound, "NOT_FOUND", "Agent not found") + return + } + if agent.Status.String != "online" { + respondError(c, http.StatusConflict, "NODE_OFFLINE", "node is offline β€” reconnect it before upgrading") + return + } + + arch := "amd64" + var caps struct { + SupportedArchitectures []string `json:"supported_architectures"` + } + unmarshalRaw(agent.Capabilities, &caps) + if len(caps.SupportedArchitectures) > 0 && caps.SupportedArchitectures[0] != "" { + arch = caps.SupportedArchitectures[0] + } + platform := "linux-" + arch // agents only ship for linux today + + dir := os.Getenv("CONTAINR_AGENT_BIN_DIR") + if dir == "" { + dir = "/agents" + } + path := filepath.Join(dir, "containr-agent-"+platform) + sum, err := sha256File(path) + if err != nil { + respondError(c, http.StatusConflict, "NO_BINARY", + fmt.Sprintf("no bundled agent binary for %s on this server", platform)) + return + } + + command, err := h.q.CreateCommand(ctx, sqlcdb.CreateCommandParams{ + ID: uuid.New().String(), + Type: "self_upgrade", + NodeAgentID: id, + Payload: rawJSON(map[string]interface{}{ + "platform": platform, + "sha256": sum, + "version": Version, + }), + }) + if err != nil { + respondError(c, http.StatusInternalServerError, "INTERNAL", "failed to enqueue upgrade") + return + } + LogAuditWithRequest(c, "node", id, "upgrade", map[string]interface{}{"platform": platform, "version": Version}) + c.JSON(http.StatusAccepted, gin.H{"command": command, "platform": platform, "version": Version}) +} + +func sha256File(path string) (string, error) { + f, err := os.Open(path) + if err != nil { + return "", err + } + defer f.Close() + h := sha256.New() + if _, err := io.Copy(h, f); err != nil { + return "", err + } + return hex.EncodeToString(h.Sum(nil)), nil +} + // ExecuteCommand executes a command on an agent func (h *NodeAgentHandler) ExecuteCommand(c *gin.Context) { agentID := c.Param("id") @@ -1390,6 +1463,7 @@ func (h *NodeAgentHandler) SetupAdminRoutes(router *gin.RouterGroup) { agents.POST("/:id/cordon", h.CordonAgent) agents.POST("/:id/uncordon", h.UncordonAgent) agents.POST("/:id/drain", h.DrainAgent) + agents.POST("/:id/upgrade", h.UpgradeAgent) agents.POST("/:id/prune", h.PruneAgent) agents.POST("/:id/commands", h.ExecuteCommand) agents.GET("/:id/commands", h.GetAgentCommands) diff --git a/app/backend/internal/cli/commands/nodes.go b/app/backend/internal/cli/commands/nodes.go index 108a3ae..8a5d7c0 100644 --- a/app/backend/internal/cli/commands/nodes.go +++ b/app/backend/internal/cli/commands/nodes.go @@ -190,6 +190,24 @@ var nodesDrainCmd = &cobra.Command{ }, } +var nodesUpgradeCmd = &cobra.Command{ + Use: "upgrade ", + Short: "Upgrade the node agent to the server's bundled binary", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + c, err := client() + if err != nil { + return err + } + data, err := c.Do("POST", "/agents/"+args[0]+"/upgrade", nil) + if err != nil { + return err + } + PrintRaw(data) + return nil + }, +} + var nodesCommandsCmd = &cobra.Command{ Use: "commands ", Short: "List recent commands sent to a node agent", @@ -277,5 +295,5 @@ func init() { nodesPruneCmd.Flags().String("until", "", "only prune objects older than this duration (e.g. 168h)") nodesPruneCmd.Flags().Bool("volumes", false, "also prune volumes (destructive β€” asks twice)") nodeTokensCmd.AddCommand(nodeTokensIssueCmd, nodeTokensListCmd, nodeTokensRevokeCmd) - NodesCmd.AddCommand(nodesListCmd, nodesGetCmd, nodesUpdateCmd, nodesCordonCmd, nodesUncordonCmd, nodesDrainCmd, nodesPruneCmd, nodesCommandsCmd, nodesDeleteCmd, nodeTokensCmd) + NodesCmd.AddCommand(nodesListCmd, nodesGetCmd, nodesUpdateCmd, nodesCordonCmd, nodesUncordonCmd, nodesDrainCmd, nodesUpgradeCmd, nodesPruneCmd, nodesCommandsCmd, nodesDeleteCmd, nodeTokensCmd) } diff --git a/app/backend/internal/mcpserver/server.go b/app/backend/internal/mcpserver/server.go index ac14b99..c77036e 100644 --- a/app/backend/internal/mcpserver/server.go +++ b/app/backend/internal/mcpserver/server.go @@ -178,6 +178,7 @@ func init() { register(spec{"containr_nodes_uncordon", "Re-open a node for placement (admin). Args: id.", "POST", "/agents/{id}/uncordon", []string{"id"}, nil, false}) register(spec{"containr_nodes_drain", "Drain a node: cordon + remove all service containers via the agent + unpin affected services so they redeploy locally. Requires the node online (admin). Args: id.", "POST", "/agents/{id}/drain", []string{"id"}, nil, true}) register(spec{"containr_nodes_prune", "Enqueue a bounded docker system prune on a node (admin). Args: id; optional until (e.g. 168h), volumes (danger, off by default).", "POST", "/agents/{id}/prune", []string{"id"}, []string{"until", "volumes"}, true}) + register(spec{"containr_nodes_upgrade", "Upgrade a node's agent to the server's bundled binary: downloads over the API, verifies sha256, swaps atomically, re-execs. Node must be online (admin). Args: id.", "POST", "/agents/{id}/upgrade", []string{"id"}, nil, true}) register(spec{"containr_nodes_commands", "List recent commands sent to a node agent (admin). Args: id.", "GET", "/agents/{id}/commands", []string{"id"}, nil, false}) // Scaling / HA / security diff --git a/app/frontend/src/features/nodes/NodeDetailPage.tsx b/app/frontend/src/features/nodes/NodeDetailPage.tsx index 1048d2d..ee6c6ee 100644 --- a/app/frontend/src/features/nodes/NodeDetailPage.tsx +++ b/app/frontend/src/features/nodes/NodeDetailPage.tsx @@ -11,6 +11,7 @@ import { listAgentContainers, pruneAgent, updateAgent, + upgradeAgent, type AgentMetricPoint, } from '@/lib/api-client'; import { getCurrentUserProfile } from '@/lib/api-client'; @@ -97,6 +98,10 @@ export function NodeDetailPage() { mutationFn: () => drainAgent(id), onSuccess: () => { setConfirmDrain(false); invalidate(); }, }); + const upgradeMutation = useMutation({ + mutationFn: () => upgradeAgent(id), + onSuccess: invalidate, + }); const agent = agentQuery.data; const metrics = useMemo(() => metricsQuery.data ?? [], [metricsQuery.data]); @@ -215,6 +220,16 @@ export function NodeDetailPage() { > drain +