feat(nodes): agent self-upgrade (#57)

POST /agents/:id/upgrade enqueues a self_upgrade command carrying the
target platform, the sha256 the server computed over its bundled binary,
and the server version. The agent downloads the replacement from the
existing /api/agents/download/:platform endpoint, verifies the checksum,
sanity-checks the new binary (--version), atomically swaps its own
executable, reports the command result, then re-execs via syscall.Exec —
no supervisor or restart plumbing required.

Agents stamp main.version via ldflags at image build, expose --version,
and now send version at register so node_agents.version refreshes on
re-register (older agents without the field keep their stored value).
The server rejects upgrades for offline nodes and for platforms with no
bundled binary (linux-amd64/linux-arm64 ship in the image).

Surfaces: nodes upgrade CLI, containr_nodes_upgrade MCP (confirm-gated),
node-detail page button, OpenAPI. Verified live: v0.0.3 agent upgraded
to v0.0.4 in place, re-registered with the new version; offline node
returns 409 NODE_OFFLINE.

Generated with [Devin](https://devin.ai)

Co-authored-by: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com>

app/backend/Dockerfile

@@ -14,8 +14,8 @@ RUN CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo \
     -ldflags "-X containr/internal/api.Version=${VERSION}" -o main cmd/server/main.go
 
 # Node-agent binaries served by the API for SSH/bootstrap installs.
-RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o /agents/containr-agent-linux-amd64 ./cmd/agent && \
-    CGO_ENABLED=0 GOOS=linux GOARCH=arm64 go build -o /agents/containr-agent-linux-arm64 ./cmd/agent
+RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -ldflags "-X main.version=${VERSION}" -o /agents/containr-agent-linux-amd64 ./cmd/agent && \
+    CGO_ENABLED=0 GOOS=linux GOARCH=arm64 go build -ldflags "-X main.version=${VERSION}" -o /agents/containr-agent-linux-arm64 ./cmd/agent
 
 # Build embedded Better Auth runtime dependencies
 FROM node:22-alpine AS auth-builder

app/backend/cmd/agent/main.go

@@ -3,14 +3,18 @@ package main
 import (
 	"bytes"
 	"context"
+	"crypto/sha256"
+	"encoding/hex"
 	"encoding/json"
 	"errors"
 	"fmt"
+	"io"
 	"log"
 	"net"
 	"net/http"
 	"os"
 	"os/exec"
+	"path/filepath"
 	"runtime"
 	"strconv"
 	"strings"
@@ -104,7 +108,15 @@ type registerResponse struct {
 	Status  string `json:"status"`
 }
 
+// version is stamped at build time via -ldflags "-X main.version=…".
+// The server reports it on upgrades so agents can show what they run.
+var version = "dev"
+
 func main() {
+	if len(os.Args) > 1 && (os.Args[1] == "--version" || os.Args[1] == "version") {
+		fmt.Printf("containr-agent %s\n", version)
+		return
+	}
 	cfg, err := loadConfig()
 	if err != nil {
 		log.Fatal(err)
@@ -167,6 +179,7 @@ func register(ctx context.Context, client *http.Client, cfg agentConfig) (string
 		"ip_address":   cfg.IP,
 		"port":         cfg.Port,
 		"capabilities": detectCapabilities(),
+		"version":      firstEnv("CONTAINR_AGENT_VERSION", version),
 		// Always send mesh so a removed interface clears stored state.
 		"mesh": detectMeshInterfaces(),
 	}
@@ -190,7 +203,7 @@ func sendHeartbeat(ctx context.Context, client *http.Client, cfg agentConfig, ag
 		"container_count": dockerContainerCount(),
 		"system_load":     readSystemLoad(),
 		"uptime":          readSystemUptime(),
-		"version":         firstEnv("CONTAINR_AGENT_VERSION", "dev"),
+		"version":         firstEnv("CONTAINR_AGENT_VERSION", version),
 	}
 	return postJSON(ctx, client, cfg, "/api/agents/heartbeat", payload, nil)
 }
@@ -216,7 +229,13 @@ func processCommands(ctx context.Context, client *http.Client, cfg agentConfig,
 		return err
 	}
 	for _, item := range list.Commands {
-		result, runErr := executeCommand(item)
+		var result, reexecPath string
+		var runErr error
+		if item.Type == "self_upgrade" {
+			result, reexecPath, runErr = selfUpgrade(cfg, item.Payload)
+		} else {
+			result, runErr = executeCommand(item)
+		}
 		status := "completed"
 		errText := ""
 		if runErr != nil {
@@ -232,6 +251,12 @@ func processCommands(ctx context.Context, client *http.Client, cfg agentConfig,
 		if err := postJSON(ctx, client, cfg, path, payload, nil); err != nil {
 			log.Printf("report command result failed: command=%s err=%v", item.ID, err)
 		}
+		// Re-exec AFTER the result posts — the new binary picks up the loop.
+		if reexecPath != "" {
+			if err := syscall.Exec(reexecPath, os.Args, os.Environ()); err != nil {
+				log.Fatalf("re-exec after upgrade failed: %v", err)
+			}
+		}
 	}
 	return nil
 }
@@ -257,6 +282,67 @@ func executeCommand(item command) (string, error) {
 	}
 }
 
+// selfUpgrade downloads the agent binary the server bundles for this
+// platform, verifies it, swaps it in atomically, and returns the path to
+// re-exec. On any failure the running binary is left untouched.
+func selfUpgrade(cfg agentConfig, payload map[string]interface{}) (string, string, error) {
+	platform := firstString(stringValue(payload["platform"]), runtime.GOOS+"-"+runtime.GOARCH)
+
+	exe, err := os.Executable()
+	if err != nil {
+		return "", "", fmt.Errorf("resolve own path: %w", err)
+	}
+	if resolved, err := filepath.EvalSymlinks(exe); err == nil {
+		exe = resolved
+	}
+
+	url := fmt.Sprintf("%s/api/agents/download/%s", cfg.APIURL, platform)
+	req, err := http.NewRequest(http.MethodGet, url, nil)
+	if err != nil {
+		return "", "", err
+	}
+	resp, err := (&http.Client{Timeout: 2 * time.Minute}).Do(req)
+	if err != nil {
+		return "", "", fmt.Errorf("download %s: %w", platform, err)
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != http.StatusOK {
+		return "", "", fmt.Errorf("download %s: server returned %d", platform, resp.StatusCode)
+	}
+
+	newPath := exe + ".new"
+	out, err := os.OpenFile(newPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o755)
+	if err != nil {
+		return "", "", fmt.Errorf("write %s: %w", newPath, err)
+	}
+	h := sha256.New()
+	if _, err := io.Copy(io.MultiWriter(out, h), resp.Body); err != nil {
+		out.Close()
+		os.Remove(newPath)
+		return "", "", fmt.Errorf("download write: %w", err)
+	}
+	out.Close()
+
+	if want := stringValue(payload["sha256"]); want != "" {
+		if got := hex.EncodeToString(h.Sum(nil)); got != want {
+			os.Remove(newPath)
+			return "", "", fmt.Errorf("sha256 mismatch: got %s want %s", got, want)
+		}
+	}
+
+	// Sanity: the new binary must at least report its version.
+	if verOut, err := exec.Command(newPath, "--version").CombinedOutput(); err != nil {
+		os.Remove(newPath)
+		return "", "", fmt.Errorf("new binary failed sanity check: %s", strings.TrimSpace(string(verOut)))
+	}
+
+	if err := os.Rename(newPath, exe); err != nil {
+		os.Remove(newPath)
+		return "", "", fmt.Errorf("swap binary: %w", err)
+	}
+	return fmt.Sprintf("upgraded to %s (%s)", firstString(stringValue(payload["version"]), "latest"), platform), exe, nil
+}
+
 // dockerPrune runs a bounded docker system prune. Volumes are excluded
 // unless the payload explicitly opts in — a stray volume prune destroys
 // database data.

app/backend/internal/api/agents.go

@@ -11,6 +11,7 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"io"
 	"log"
 	"math"
 	"net/http"
@@ -353,6 +354,7 @@ func (h *NodeAgentHandler) RegisterAgent(c *gin.Context) {
 		IPAddress    string            `json:"ip_address" binding:"required"`
 		Port         int               `json:"port" binding:"required"`
 		Capabilities AgentCapabilities `json:"capabilities" binding:"required"`
+		Version      string            `json:"version"`
 		AuthToken    string            `json:"auth_token"`
 		Mesh         map[string]string `json:"mesh"`
 	}
@@ -392,13 +394,14 @@ func (h *NodeAgentHandler) RegisterAgent(c *gin.Context) {
 			}
 		}
 		updated, err := h.q.UpdateAgent(ctx, sqlcdb.UpdateAgentParams{
-			ID:            existing.ID,
-			Name:          req.Name,
-			Hostname:      existing.Hostname,
-			IpAddress:     existing.IpAddress,
-			Port:          int32(req.Port),
-			Status:        sql.NullString{String: "connecting", Valid: true},
-			Version:       existing.Version,
+			ID:        existing.ID,
+			Name:      req.Name,
+			Hostname:  existing.Hostname,
+			IpAddress: existing.IpAddress,
+			Port:      int32(req.Port),
+			Status:    sql.NullString{String: "connecting", Valid: true},
+			// Older agents don't report a version — keep whatever's stored.
+			Version:       sql.NullString{String: firstNonEmpty(req.Version, existing.Version.String), Valid: true},
 			Capabilities:  rawJSON(req.Capabilities),
 			Resources:     existing.Resources,
 			LastHeartbeat: sql.NullTime{Time: time.Now(), Valid: true},
@@ -450,7 +453,7 @@ func (h *NodeAgentHandler) RegisterAgent(c *gin.Context) {
 		IpAddress:     req.IPAddress,
 		Port:          int32(req.Port),
 		Status:        sql.NullString{String: "connecting", Valid: true},
-		Version:       sql.NullString{},
+		Version:       sql.NullString{String: req.Version, Valid: req.Version != ""},
 		Capabilities:  rawJSON(req.Capabilities),
 		Resources:     rawJSON(resources),
 		LastHeartbeat: sql.NullTime{Time: time.Now(), Valid: true},
@@ -911,6 +914,76 @@ func (h *NodeAgentHandler) PruneAgent(c *gin.Context) {
 	c.JSON(http.StatusAccepted, gin.H{"command": command})
 }
 
+// UpgradeAgent enqueues a self_upgrade command: the agent downloads its
+// replacement binary from this server's bundled /download endpoint,
+// verifies the sha256, swaps it in, and re-execs. Fails fast when no
+// bundled binary exists for the agent's architecture.
+func (h *NodeAgentHandler) UpgradeAgent(c *gin.Context) {
+	ctx := c.Request.Context()
+	id := c.Param("id")
+	agent, err := h.q.GetAgent(ctx, id)
+	if err != nil {
+		respondError(c, http.StatusNotFound, "NOT_FOUND", "Agent not found")
+		return
+	}
+	if agent.Status.String != "online" {
+		respondError(c, http.StatusConflict, "NODE_OFFLINE", "node is offline — reconnect it before upgrading")
+		return
+	}
+
+	arch := "amd64"
+	var caps struct {
+		SupportedArchitectures []string `json:"supported_architectures"`
+	}
+	unmarshalRaw(agent.Capabilities, &caps)
+	if len(caps.SupportedArchitectures) > 0 && caps.SupportedArchitectures[0] != "" {
+		arch = caps.SupportedArchitectures[0]
+	}
+	platform := "linux-" + arch // agents only ship for linux today
+
+	dir := os.Getenv("CONTAINR_AGENT_BIN_DIR")
+	if dir == "" {
+		dir = "/agents"
+	}
+	path := filepath.Join(dir, "containr-agent-"+platform)
+	sum, err := sha256File(path)
+	if err != nil {
+		respondError(c, http.StatusConflict, "NO_BINARY",
+			fmt.Sprintf("no bundled agent binary for %s on this server", platform))
+		return
+	}
+
+	command, err := h.q.CreateCommand(ctx, sqlcdb.CreateCommandParams{
+		ID:          uuid.New().String(),
+		Type:        "self_upgrade",
+		NodeAgentID: id,
+		Payload: rawJSON(map[string]interface{}{
+			"platform": platform,
+			"sha256":   sum,
+			"version":  Version,
+		}),
+	})
+	if err != nil {
+		respondError(c, http.StatusInternalServerError, "INTERNAL", "failed to enqueue upgrade")
+		return
+	}
+	LogAuditWithRequest(c, "node", id, "upgrade", map[string]interface{}{"platform": platform, "version": Version})
+	c.JSON(http.StatusAccepted, gin.H{"command": command, "platform": platform, "version": Version})
+}
+
+func sha256File(path string) (string, error) {
+	f, err := os.Open(path)
+	if err != nil {
+		return "", err
+	}
+	defer f.Close()
+	h := sha256.New()
+	if _, err := io.Copy(h, f); err != nil {
+		return "", err
+	}
+	return hex.EncodeToString(h.Sum(nil)), nil
+}
+
 // ExecuteCommand executes a command on an agent
 func (h *NodeAgentHandler) ExecuteCommand(c *gin.Context) {
 	agentID := c.Param("id")
@@ -1390,6 +1463,7 @@ func (h *NodeAgentHandler) SetupAdminRoutes(router *gin.RouterGroup) {
 		agents.POST("/:id/cordon", h.CordonAgent)
 		agents.POST("/:id/uncordon", h.UncordonAgent)
 		agents.POST("/:id/drain", h.DrainAgent)
+		agents.POST("/:id/upgrade", h.UpgradeAgent)
 		agents.POST("/:id/prune", h.PruneAgent)
 		agents.POST("/:id/commands", h.ExecuteCommand)
 		agents.GET("/:id/commands", h.GetAgentCommands)

app/backend/internal/cli/commands/nodes.go

@@ -190,6 +190,24 @@ var nodesDrainCmd = &cobra.Command{
 	},
 }
 
+var nodesUpgradeCmd = &cobra.Command{
+	Use:   "upgrade <id>",
+	Short: "Upgrade the node agent to the server's bundled binary",
+	Args:  cobra.ExactArgs(1),
+	RunE: func(cmd *cobra.Command, args []string) error {
+		c, err := client()
+		if err != nil {
+			return err
+		}
+		data, err := c.Do("POST", "/agents/"+args[0]+"/upgrade", nil)
+		if err != nil {
+			return err
+		}
+		PrintRaw(data)
+		return nil
+	},
+}
+
 var nodesCommandsCmd = &cobra.Command{
 	Use:   "commands <id>",
 	Short: "List recent commands sent to a node agent",
@@ -277,5 +295,5 @@ func init() {
 	nodesPruneCmd.Flags().String("until", "", "only prune objects older than this duration (e.g. 168h)")
 	nodesPruneCmd.Flags().Bool("volumes", false, "also prune volumes (destructive — asks twice)")
 	nodeTokensCmd.AddCommand(nodeTokensIssueCmd, nodeTokensListCmd, nodeTokensRevokeCmd)
-	NodesCmd.AddCommand(nodesListCmd, nodesGetCmd, nodesUpdateCmd, nodesCordonCmd, nodesUncordonCmd, nodesDrainCmd, nodesPruneCmd, nodesCommandsCmd, nodesDeleteCmd, nodeTokensCmd)
+	NodesCmd.AddCommand(nodesListCmd, nodesGetCmd, nodesUpdateCmd, nodesCordonCmd, nodesUncordonCmd, nodesDrainCmd, nodesUpgradeCmd, nodesPruneCmd, nodesCommandsCmd, nodesDeleteCmd, nodeTokensCmd)
 }

app/backend/internal/mcpserver/server.go

@@ -178,6 +178,7 @@ func init() {
 	register(spec{"containr_nodes_uncordon", "Re-open a node for placement (admin). Args: id.", "POST", "/agents/{id}/uncordon", []string{"id"}, nil, false})
 	register(spec{"containr_nodes_drain", "Drain a node: cordon + remove all service containers via the agent + unpin affected services so they redeploy locally. Requires the node online (admin). Args: id.", "POST", "/agents/{id}/drain", []string{"id"}, nil, true})
 	register(spec{"containr_nodes_prune", "Enqueue a bounded docker system prune on a node (admin). Args: id; optional until (e.g. 168h), volumes (danger, off by default).", "POST", "/agents/{id}/prune", []string{"id"}, []string{"until", "volumes"}, true})
+	register(spec{"containr_nodes_upgrade", "Upgrade a node's agent to the server's bundled binary: downloads over the API, verifies sha256, swaps atomically, re-execs. Node must be online (admin). Args: id.", "POST", "/agents/{id}/upgrade", []string{"id"}, nil, true})
 	register(spec{"containr_nodes_commands", "List recent commands sent to a node agent (admin). Args: id.", "GET", "/agents/{id}/commands", []string{"id"}, nil, false})
 
 	// Scaling / HA / security