feat: build-on-node — git checkouts, remote image builds, image transport (#60)

Git-sourced deploys never materialized a checkout: sourcePath was
build_path resolved against the server cwd, so builds consumed the
backend's own directory. Remote placement could only run
registry-pulled images.

- internal/source: per-deployment checkout (init/fetch/checkout) at
  branch or commit; GIT_ASKPASS keeps tokens out of argv, .git/config,
  and the build context; dumb-transport and SHA-refusal fallbacks;
  build_path validated inside the checkout.
- BuildManager.PackageContext: produces the same tar context Build
  ships to the daemon — generated Dockerfile at tar root, .git
  excluded. Fixes nixpacks passing the Dockerfile path as content.
- Agent artifact endpoint: GET /api/agents/artifacts/:id streams
  build contexts and docker-save tarballs behind agent-token auth;
  hex-validated ids, per-deployment cleanup, boot-time purge.
- Agent commands: build_image (docker build -, --build-arg values via
  process env, not argv) and load_image (docker load for containr-*
  rollback tags no registry serves).
- Deployment flow: remote git builds package context → artifact →
  node docker-builds the deployment tag; remote rollbacks of local
  tags export via SaveImage → artifact → node docker-loads.

Verified end-to-end: agent-pinned git service built on the node
(build_image → create_container → deployed) and rollback shipped a
docker-save tarball (load_image → deployed). Artifact auth returns
401/404 correctly; workspaces and blobs are removed after deploy.

Authored By: TDvorak <[email protected]>

app/backend/internal/api/agents.go

@@ -769,7 +769,9 @@ func (h *NodeAgentHandler) CompleteCommand(c *gin.Context) {
 }
 
 // scrubCommandSecrets rewrites a completed command's payload with any
-// registry password blanked. Best-effort — a failed scrub leaves the row.
+// credential-bearing fields blanked — registry passwords and build-time
+// args (tokens like NPM_TOKEN ride along as build args). Best-effort — a
+// failed scrub leaves the row.
 func scrubCommandSecrets(ctx context.Context, q *sqlcdb.Queries, commandID string, payload pqtype.NullRawMessage) {
 	if !payload.Valid || len(payload.RawMessage) == 0 {
 		return
@@ -778,15 +780,24 @@ func scrubCommandSecrets(ctx context.Context, q *sqlcdb.Queries, commandID strin
 	if err := json.Unmarshal(payload.RawMessage, &doc); err != nil {
 		return
 	}
-	container, ok := doc["container"].(map[string]interface{})
-	if !ok {
+	dirty := false
+	if container, ok := doc["container"].(map[string]interface{}); ok {
+		if reg, ok := container["registry"].(map[string]interface{}); ok && reg["password"] != nil {
+			reg["password"] = "***"
+			dirty = true
+		}
+	}
+	for _, key := range []string{"build_args", "environment"} {
+		if args, ok := doc[key].(map[string]interface{}); ok && len(args) > 0 {
+			for k := range args {
+				args[k] = "***"
+			}
+			dirty = true
+		}
+	}
+	if !dirty {
 		return
 	}
-	reg, ok := container["registry"].(map[string]interface{})
-	if !ok || reg["password"] == nil {
-		return
-	}
-	reg["password"] = "***"
 	if err := q.ScrubCommandPayload(ctx, sqlcdb.ScrubCommandPayloadParams{
 		ID:      commandID,
 		Payload: rawJSON(doc),
@@ -1458,6 +1469,10 @@ func (h *NodeAgentHandler) SetupPublicRoutes(router *gin.RouterGroup) {
 		// public; the enroll token is the actual secret.
 		agents.GET("/install.sh", h.ServeInstallScript)
 		agents.GET("/download/:platform", h.ServeAgentBinary)
+
+		// Deploy artifacts (build contexts, image tarballs) — token-gated;
+		// the blobs are per-deployment source code, not public bootstrap.
+		agents.GET("/artifacts/:id", h.ServeArtifact)
 	}
 }
 

app/backend/internal/api/artifacts.go

@@ -0,0 +1,136 @@
+package api
+
+import (
+	"crypto/rand"
+	"encoding/hex"
+	"io"
+	"net/http"
+	"os"
+	"path/filepath"
+	"sync"
+	"time"
+
+	"github.com/gin-gonic/gin"
+)
+
+// containrWorkDir is the build workspace root — same dir the BuildManager
+// was constructed with; checkouts and agent artifacts live beneath it.
+const containrWorkDir = "/tmp/containr-builds"
+
+// artifacts is a one-shot blob store for node agents: build contexts and
+// docker-save tarballs the server produced on behalf of a deployment. Files
+// live under workdir/artifacts/<id> and are removed when the deployment
+// finishes or after maxArtifactAge.
+var artifacts = &artifactStore{files: map[string]string{}}
+
+type artifactStore struct {
+	mu    sync.Mutex
+	dir   string
+	files map[string]string
+}
+
+const maxArtifactAge = 2 // hours — a deployment can never outlive this
+
+// initArtifacts points the store at dir (created on demand).
+func initArtifacts(dir string) {
+	artifacts.mu.Lock()
+	defer artifacts.mu.Unlock()
+	artifacts.dir = dir
+	_ = os.MkdirAll(dir, 0o755)
+}
+
+// artifactDir returns the store root, defaulting under the build workdir.
+func artifactDir() string {
+	artifacts.mu.Lock()
+	defer artifacts.mu.Unlock()
+	if artifacts.dir != "" {
+		return artifacts.dir
+	}
+	return filepath.Join(containrWorkDir, "artifacts")
+}
+
+// putArtifact streams rc into the store and returns the artifact id.
+func putArtifact(rc io.Reader) (string, error) {
+	buf := make([]byte, 16)
+	if _, err := rand.Read(buf); err != nil {
+		return "", err
+	}
+	id := hex.EncodeToString(buf)
+	dir := artifactDir()
+	if err := os.MkdirAll(dir, 0o755); err != nil {
+		return "", err
+	}
+	path := filepath.Join(dir, id+".tar")
+	f, err := os.Create(path)
+	if err != nil {
+		return "", err
+	}
+	if _, err := io.Copy(f, rc); err != nil {
+		f.Close()
+		os.Remove(path)
+		return "", err
+	}
+	f.Close()
+
+	artifacts.mu.Lock()
+	artifacts.files[id] = path
+	artifacts.mu.Unlock()
+	return id, nil
+}
+
+// removeArtifact deletes the file and forgets the id. Safe on missing ids.
+func removeArtifact(id string) {
+	artifacts.mu.Lock()
+	path, ok := artifacts.files[id]
+	delete(artifacts.files, id)
+	artifacts.mu.Unlock()
+	if ok {
+		_ = os.Remove(path)
+		return
+	}
+	// Registered before a restart — remove best-effort by name.
+	if id != "" {
+		_ = os.Remove(filepath.Join(artifactDir(), id+".tar"))
+	}
+}
+
+// ServeArtifact streams a stored blob to a token-authenticated agent. Files
+// persist across downloads — spread builds share one context — and are
+// deleted by the deployment that created them.
+func (h *NodeAgentHandler) ServeArtifact(c *gin.Context) {
+	if !h.isValidAgentAuthToken(c.Request.Context(), agentAuthTokenFromRequest(c)) {
+		c.JSON(http.StatusUnauthorized, gin.H{"error": "Invalid auth token"})
+		return
+	}
+	id := c.Param("id")
+	raw, err := hex.DecodeString(id)
+	if len(id) != 32 || err != nil || len(raw) != 16 {
+		c.JSON(http.StatusNotFound, gin.H{"error": "Artifact not found"})
+		return
+	}
+	path := filepath.Join(artifactDir(), id+".tar")
+	info, err := os.Stat(path)
+	if err != nil || info.IsDir() {
+		c.JSON(http.StatusNotFound, gin.H{"error": "Artifact not found"})
+		return
+	}
+	c.Header("Content-Type", "application/x-tar")
+	c.File(path)
+}
+
+// purgeStaleArtifacts drops files older than maxArtifactAge. Called on boot.
+func purgeStaleArtifacts() {
+	entries, err := os.ReadDir(artifactDir())
+	if err != nil {
+		return
+	}
+	for _, e := range entries {
+		info, err := e.Info()
+		if err != nil {
+			continue
+		}
+		if info.ModTime().Before(time.Now().Add(-maxArtifactAge * time.Hour)) {
+			_ = os.Remove(filepath.Join(artifactDir(), e.Name()))
+		}
+	}
+}

app/backend/internal/api/deployments.go

@@ -5,11 +5,15 @@ import (
 	"containr/internal/deployment"
 	"containr/internal/deployqueue"
 	"containr/internal/docker"
+	"containr/internal/source"
+	"containr/internal/types"
 	"context"
 	"errors"
 	"fmt"
 	"log/slog"
 	"net/http"
+	"os"
+	"path/filepath"
 	"strings"
 	"time"
 
@@ -377,6 +381,23 @@ func runDeploymentAndSyncWithImage(
 	ctx, cancel := context.WithTimeout(parentCtx, 30*time.Minute)
 	defer cancel()
 
+	// The checkout and any shipped artifacts die with this deployment.
+	var artifactIDs []string
+	checkoutRoot := filepath.Join(containrWorkDir, "repos", dbDeployment.ID.String())
+	defer func() {
+		_ = os.RemoveAll(checkoutRoot)
+		for _, id := range artifactIDs {
+			removeArtifact(id)
+		}
+	}()
+	failDeploy := func(msg string) {
+		failedAt := time.Now()
+		_, _ = db.Exec(
+			`UPDATE deployments SET status = 'failed', error = $1, completed_at = $2, updated_at = $2 WHERE id = $3`,
+			msg, failedAt, dbDeployment.ID,
+		)
+	}
+
 	sourcePath := strings.TrimSpace(service.BuildPath)
 	if sourcePath == "" {
 		sourcePath = "."
@@ -421,19 +442,31 @@ func runDeploymentAndSyncWithImage(
 		maintURL = maintenanceURL(db)
 	}
 
-	// Node-pinned services dispatch to the agent's command queue. Builds run
-	// on the local host, so remote placement only works for registry-pulled
-	// images — git-sourced and rollback deploys fail fast with that message.
+	// Node-pinned and spread services dispatch to agent command queues.
 	nodeID := serviceNodeID(db, service.ID)
 	spread := serviceSpread(db, service.ID)
-	if (nodeID != "" || spread) && (service.GitRepo != "" || imageOverride != "") {
-		failedAt := time.Now()
-		msg := "service uses remote node placement but remote nodes only run registry-pulled images (git builds and rollbacks stay local — build-on-node is not yet supported)"
-		_, _ = db.Exec(
-			`UPDATE deployments SET status = 'failed', error = $1, completed_at = $2, updated_at = $2 WHERE id = $3`,
-			msg, failedAt, dbDeployment.ID,
-		)
-		return
+	remote := nodeID != "" || spread
+
+	// Materialize the git checkout — local builds consume it directly;
+	// remote builds package it into a context artifact the node builds.
+	if service.GitRepo != "" && imageOverride == "" {
+		var ownerID string
+		_ = db.QueryRow(`SELECT owner_id FROM projects WHERE id = $1`, service.ProjectID).Scan(&ownerID)
+		branch := req.Branch
+		if branch == "" {
+			branch = service.GitBranch
+		}
+		cloneSpec, err := resolveGitCloneSpec(db, ownerID, service.GitRepo, branch, req.CommitHash, service.BuildPath)
+		if err != nil {
+			failDeploy("resolve git source: " + err.Error())
+			return
+		}
+		src, err := source.Checkout(ctx, checkoutRoot, cloneSpec)
+		if err != nil {
+			failDeploy("git checkout: " + err.Error())
+			return
+		}
+		sourcePath = src
 	}
 
 	deployReq := &deployment.DeploymentRequest{
@@ -480,14 +513,36 @@ func runDeploymentAndSyncWithImage(
 		if image == "" {
 			image = service.Image
 		}
-		var ownerID string
-		_ = db.QueryRow(`SELECT owner_id FROM projects WHERE id = $1`, service.ProjectID).Scan(&ownerID)
-		auth := registryAuthFor(db, ownerID, image)
-		deployReq.BuildConfig = &deployment.BuildConfig{
-			BuildType:     "prebuilt",
-			PrebuiltImage: image,
-			PullUsername:  auth.Username,
-			PullPassword:  auth.Password,
+		if remote && strings.HasPrefix(image, "containr-") {
+			// Locally-built tag — no registry serves it. Ship the image
+			// itself as a docker-save artifact; the node loads it.
+			rc, err := engine.DockerClient().SaveImage(ctx, image)
+			if err != nil {
+				failDeploy("export rollback image: " + err.Error())
+				return
+			}
+			artifactID, err := putArtifact(rc)
+			_ = rc.Close()
+			if err != nil {
+				failDeploy("store rollback image artifact: " + err.Error())
+				return
+			}
+			artifactIDs = append(artifactIDs, artifactID)
+			deployReq.Config.RemoteLoad = &deployment.RemoteLoadSpec{ArtifactID: artifactID}
+			deployReq.BuildConfig = &deployment.BuildConfig{
+				BuildType:     "remote",
+				PrebuiltImage: image,
+			}
+		} else {
+			var ownerID string
+			_ = db.QueryRow(`SELECT owner_id FROM projects WHERE id = $1`, service.ProjectID).Scan(&ownerID)
+			auth := registryAuthFor(db, ownerID, image)
+			deployReq.BuildConfig = &deployment.BuildConfig{
+				BuildType:     "prebuilt",
+				PrebuiltImage: image,
+				PullUsername:  auth.Username,
+				PullPassword:  auth.Password,
+			}
 		}
 	} else {
 		// "auto" leaves BuildType empty so the manager detects it; the static
@@ -497,16 +552,54 @@ func runDeploymentAndSyncWithImage(
 		if service.Builder != "" && service.Builder != "auto" {
 			buildType = service.Builder
 		}
-		deployReq.BuildConfig = &deployment.BuildConfig{
-			BuildType:  buildType,
-			SourcePath: sourcePath,
-			Branch:     req.Branch,
-			Commit:     req.CommitHash,
-			NoCache:    req.NoCache,
-		}
+		buildArgs := map[string]string{}
+		buildCommand := ""
 		if buildType == "static" {
-			deployReq.BuildConfig.BuildCommand = firstNonEmpty(service.StaticBuildCmd, "npm ci && npm run build")
-			deployReq.BuildConfig.BuildArgs = map[string]string{"STATIC_DIR": firstNonEmpty(service.StaticDir, "dist")}
+			buildCommand = firstNonEmpty(service.StaticBuildCmd, "npm ci && npm run build")
+			buildArgs["STATIC_DIR"] = firstNonEmpty(service.StaticDir, "dist")
+		}
+		if remote {
+			// Package the context — the node docker-builds it itself.
+			packReq := &types.BuildRequest{
+				BuildType:    buildType,
+				SourcePath:   sourcePath,
+				BuildCommand: buildCommand,
+				BuildArgs:    buildArgs,
+				Environment:  env,
+			}
+			rc, err := engine.BuildManager().PackageContext(ctx, packReq)
+			if err != nil {
+				failDeploy("package build context: " + err.Error())
+				return
+			}
+			artifactID, err := putArtifact(rc)
+			_ = rc.Close()
+			if err != nil {
+				failDeploy("store build context artifact: " + err.Error())
+				return
+			}
+			artifactIDs = append(artifactIDs, artifactID)
+			deployReq.Config.RemoteBuild = &deployment.RemoteBuildSpec{
+				ArtifactID: artifactID,
+				BuildArgs:  buildArgs,
+				NoCache:    req.NoCache,
+			}
+			deployReq.BuildConfig = &deployment.BuildConfig{
+				BuildType: "remote",
+				Branch:    req.Branch,
+				Commit:    req.CommitHash,
+				NoCache:   req.NoCache,
+			}
+		} else {
+			deployReq.BuildConfig = &deployment.BuildConfig{
+				BuildType:    buildType,
+				SourcePath:   sourcePath,
+				BuildCommand: buildCommand,
+				BuildArgs:    buildArgs,
+				Branch:       req.Branch,
+				Commit:       req.CommitHash,
+				NoCache:      req.NoCache,
+			}
 		}
 	}
 

app/backend/internal/api/gitcreds.go

@@ -0,0 +1,84 @@
+package api
+
+import (
+	"fmt"
+	"strconv"
+	"strings"
+
+	"containr/internal/database"
+	"containr/internal/source"
+)
+
+// resolveGitCloneSpec turns a service's git_repo (clone URL, ssh remote, or
+// owner/repo) into a source.Spec with the project owner's provider token.
+// Public repos get no token — the clone proceeds anonymously. github_app
+// providers store the installation id, not a usable token; a fresh
+// installation token is minted per deploy.
+func resolveGitCloneSpec(db *database.DB, ownerID, gitRepo, branch, commit, buildPath string) (source.Spec, error) {
+	raw := strings.TrimSpace(gitRepo)
+	cloneURL, err := source.NormalizeCloneURL(raw)
+	if err != nil {
+		return source.Spec{}, err
+	}
+
+	spec := source.Spec{Branch: branch, Commit: commit, BuildPath: buildPath}
+
+	var providerID string
+	if !strings.Contains(cloneURL, "://") {
+		// Bare owner/repo — the connected repository row knows the host.
+		err = db.QueryRow(
+			`SELECT clone_url, provider_id FROM git_repositories
+			 WHERE full_name = $1 AND user_id = $2 LIMIT 1`,
+			cloneURL, ownerID,
+		).Scan(&cloneURL, &providerID)
+		if err != nil {
+			return source.Spec{}, fmt.Errorf("repository %s is not connected — connect it under Git settings or store a clone URL", raw)
+		}
+	} else {
+		// URL form — match a connected repo for its provider when possible;
+		// an unmatched public URL still clones, just without credentials.
+		_ = db.QueryRow(
+			`SELECT provider_id FROM git_repositories
+			 WHERE clone_url = $1 AND user_id = $2 LIMIT 1`,
+			cloneURL, ownerID,
+		).Scan(&providerID)
+	}
+
+	if err := source.ValidateRemoteURL(cloneURL); err != nil {
+		return source.Spec{}, err
+	}
+	spec.CloneURL = cloneURL
+
+	if providerID == "" {
+		return spec, nil
+	}
+	var name, apiURL, token string
+	if err := db.QueryRow(
+		`SELECT name, api_url, access_token FROM git_providers
+		 WHERE id = $1 AND user_id = $2`,
+		providerID, ownerID,
+	).Scan(&name, &apiURL, &token); err != nil {
+		return spec, nil // provider row gone — anonymous clone still works for public repos
+	}
+
+	switch strings.ToLower(name) {
+	case "github_app":
+		installationID, err := strconv.ParseInt(strings.TrimSpace(token), 10, 64)
+		if err == nil && installationID > 0 {
+			if minted, err := getGitHubAppInstallationToken(apiURL, installationID); err == nil {
+				spec.Token = minted
+				spec.User = "x-access-token"
+			}
+		}
+	case "gitlab":
+		spec.Token = token
+		spec.User = "oauth2"
+	case "bitbucket":
+		spec.Token = token
+		spec.User = "x-token-auth"
+	default:
+		spec.Token = token
+		spec.User = "x-access-token"
+	}
+	return spec, nil
+}

app/backend/internal/api/noderunner.go

@@ -157,6 +157,45 @@ func (r *agentNodeRunner) reconcileOnAgents(ctx context.Context, spec deployment
 		})
 	}
 
+	// Ship the image to every distinct target agent before replicas start.
+	// RemoteBuild nodes docker-build from the packaged context artifact;
+	// RemoteLoad nodes docker-load a saved tarball (local-tag rollbacks).
+	// Registry-pullable images skip this — docker run pulls on demand.
+	if spec.RemoteBuild != nil || spec.RemoteLoad != nil {
+		seen := map[string]bool{}
+		for i := 0; i < replicas; i++ {
+			agentID := targetFor(i)
+			if seen[agentID] {
+				continue
+			}
+			seen[agentID] = true
+			if spec.RemoteBuild != nil {
+				buildArgs := map[string]interface{}{}
+				for k, v := range spec.RemoteBuild.BuildArgs {
+					buildArgs[k] = v
+				}
+				_, err := r.enqueueAndWaitTimeout(ctx, agentID, "", "build_image",
+					map[string]interface{}{
+						"artifact_id": spec.RemoteBuild.ArtifactID,
+						"image":       spec.Image,
+						"build_args":  buildArgs,
+						"no_cache":    spec.RemoteBuild.NoCache,
+					}, 25*time.Minute)
+				if err != nil {
+					return nil, fmt.Errorf("build image on node %s: %w", agentID, err)
+				}
+			} else if spec.RemoteLoad != nil {
+				_, err := r.enqueueAndWaitTimeout(ctx, agentID, "", "load_image",
+					map[string]interface{}{
+						"artifact_id": spec.RemoteLoad.ArtifactID,
+					}, 10*time.Minute)
+				if err != nil {
+					return nil, fmt.Errorf("load image on node %s: %w", agentID, err)
+				}
+			}
+		}
+	}
+
 	state := &deployment.RuntimeState{Desired: replicas, Status: "running"}
 	for i := 0; i < replicas; i++ {
 		agentID := targetFor(i)
@@ -375,6 +414,10 @@ func (r *agentNodeRunner) RetireService(ctx context.Context, serviceID string) e
 // poll on their own interval (≥5s), so this blocks — it runs inside the
 // deploy queue worker, never on a request path.
 func (r *agentNodeRunner) enqueueAndWait(ctx context.Context, agentID, containerID, cmdType string, payload map[string]interface{}) (string, error) {
+	return r.enqueueAndWaitTimeout(ctx, agentID, containerID, cmdType, payload, 8*time.Minute)
+}
+
+func (r *agentNodeRunner) enqueueAndWaitTimeout(ctx context.Context, agentID, containerID, cmdType string, payload map[string]interface{}, timeout time.Duration) (string, error) {
 	var cid sql.NullString
 	if containerID != "" {
 		cid = sql.NullString{String: containerID, Valid: true}
@@ -390,7 +433,7 @@ func (r *agentNodeRunner) enqueueAndWait(ctx context.Context, agentID, container
 		return "", err
 	}
 
-	deadline := time.Now().Add(8 * time.Minute) // image pulls on a cold node
+	deadline := time.Now().Add(timeout) // image pulls/builds on a cold node
 	for {
 		row, err := r.q.GetCommandForAgent(ctx, sqlcdb.GetCommandForAgentParams{
 			ID:          cmd.ID,

app/backend/internal/api/routes.go

@@ -5,6 +5,7 @@ import (
 	"log"
 	"net/http"
 	"os"
+	"path/filepath"
 	"time"
 
 	"containr/internal/build"
@@ -43,6 +44,8 @@ func SetupRoutes(router *gin.Engine, db *database.DB, redis *database.Redis, cfg
 		dockerClient = client
 		buildManager = build.NewBuildManager("/tmp/containr-builds", dockerClient)
 		deploymentEngine = deployment.NewDeploymentEngine(buildManager, dockerClient)
+		initArtifacts(filepath.Join("/tmp/containr-builds", "artifacts"))
+		purgeStaleArtifacts()
 		// Node-pinned services dispatch container lifecycle to the agent's
 		// command queue instead of the local Docker socket.
 		deploymentEngine.SetNodeRunner(newAgentNodeRunner(db))