feat(scheduling): multi-node service placement + remote lifecycle (#54)

* feat(scheduling): multi-node service placement + remote container lifecycle

services.node_id pins a service to a node agent: explicit id, "auto"
(least-loaded online node), or "local"/empty for the local Docker host.

- DeploymentEngine gains a NodeRunner seam; ReconcileService routes
  pinned services to the agent's command queue (create_container with
  image/env/command/ports/volumes/labels/resource limits), tracked in
  container_instances with deterministic per-replica ids.
- Stop/start/restart/delete fan out to the pinned node — no more leaked
  remote containers on service delete.
- Agent create_container is idempotent (rm -f first) and honors restart
  policy, labels, memory/cpus, published+ephemeral ports, volumes, env.
- GET /nodes/options exposes id/name/status to authed users; full agent
  management stays admin-only.
- CLI services create/update --node, MCP node_id docs, OpenAPI
  node_id/node_name + /nodes/options, service-detail Placement section.

Ceilings: remote nodes run registry-pulled images only — git builds,
Traefik domains, and remote sleep detection stay local until
build-on-node and per-node ingress land.

Generated with [Devin](https://devin.ai)

Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* feat(scheduling): node cordon/drain + schedulable gate

- node_agents.schedulable column; "auto" placement and explicit pins skip
  cordoned nodes; already-pinned services keep running (reconcile is
  unaffected — cordon is not eviction).
- POST /agents/:id/cordon|uncordon|drain (admin). Drain cordons, removes
  every service container via the agent queue, and clears the services'
  pins so they redeploy on the local host. Requires the node online —
  an offline node can't acknowledge removals.
- CLI nodes cordon|uncordon|drain (drain confirm-gated), MCP
  containr_nodes_cordon|uncordon|drain, OpenAPI paths + schedulable
  field, node-detail page cordon badge + drain/cordon controls.
- Fix: re-registering an agent no longer resets auto_prune.

Generated with [Devin](https://devin.ai)

Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* feat(scheduling): remote registry pulls + post-run health check

- Dispatch payload carries the project owner's registry credentials for
  the image host; the agent docker-logins for the pull and logs out
  after, so private images deploy to remote nodes.
- Completed command payloads get their registry password scrubbed —
  secrets don't linger in agent_commands rows.
- Agent verifies the container is still running ~1.5s after docker run;
  crash-loops now fail the deployment with the container's last log
  lines instead of reporting "running" optimistically.

Generated with [Devin](https://devin.ai)

Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com>

---------

Co-authored-by: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com>

app/backend/internal/api/runtime.go

@@ -18,6 +18,38 @@ import (
 	"github.com/google/uuid"
 )
 
+// serviceLifecycle routes a start/stop/restart action to the node agent when
+// the service is pinned, or to the local Docker engine otherwise. Remote
+// actions block on the agent's poll loop; that matches the synchronous
+// behavior the local path already has.
+func serviceLifecycle(ctx context.Context, db *database.DB, engine *deployment.DeploymentEngine, serviceID uuid.UUID, action string) error {
+	if nodeID := serviceNodeID(db, serviceID); nodeID != "" {
+		return engine.ControlServiceOnNode(ctx, serviceID.String(), nodeID, action)
+	}
+	switch action {
+	case "start":
+		return engine.StartService(ctx, serviceID.String())
+	case "stop":
+		return engine.StopService(ctx, serviceID.String())
+	case "restart":
+		if err := engine.StopService(ctx, serviceID.String()); err != nil {
+			return err
+		}
+		return engine.StartService(ctx, serviceID.String())
+	}
+	return fmt.Errorf("unsupported lifecycle action %q", action)
+}
+
+// removeServiceRuntime tears down replicas wherever they run — remote node
+// when pinned, local Docker otherwise.
+func removeServiceRuntime(ctx context.Context, db *database.DB, engine *deployment.DeploymentEngine, serviceID uuid.UUID) {
+	if nodeID := serviceNodeID(db, serviceID); nodeID != "" {
+		_ = engine.RemoveServiceContainersOnNode(ctx, serviceID.String(), nodeID)
+		return
+	}
+	_ = engine.RemoveServiceContainers(ctx, serviceID.String())
+}
+
 // serviceRuntimeSpec builds the container spec for a service: stored env vars
 // merged with ${{Service.KEY}} references resolved across the project.
 func serviceRuntimeSpec(db *database.DB, service Service) (deployment.RuntimeSpec, error) {
@@ -37,6 +69,7 @@ func serviceRuntimeSpec(db *database.DB, service Service) (deployment.RuntimeSpe
 		Domain:        service.Domain,
 		HealthPath:    service.HealthCheckPath,
 		RestartPolicy: service.RestartPolicy,
+		NodeID:        serviceNodeID(db, service.ID),
 	}
 	// Best effort: reuse the host port from the last live deployment so the
 	// public URL survives restarts. Column exists post-migration; older DBs
@@ -384,7 +417,7 @@ func handleServiceStart(c *gin.Context) {
 	if !ok {
 		return
 	}
-	if err := engine.StartService(c.Request.Context(), service.ID.String()); err != nil {
+	if err := serviceLifecycle(c.Request.Context(), db, engine, service.ID, "start"); err != nil {
 		c.JSON(http.StatusBadGateway, gin.H{"error": err.Error()})
 		return
 	}
@@ -402,7 +435,7 @@ func handleServiceStop(c *gin.Context) {
 	if !ok {
 		return
 	}
-	if err := engine.StopService(c.Request.Context(), service.ID.String()); err != nil {
+	if err := serviceLifecycle(c.Request.Context(), db, engine, service.ID, "stop"); err != nil {
 		c.JSON(http.StatusBadGateway, gin.H{"error": err.Error()})
 		return
 	}
@@ -421,11 +454,7 @@ func handleServiceRestart(c *gin.Context) {
 		return
 	}
 	ctx := c.Request.Context()
-	if err := engine.StopService(ctx, service.ID.String()); err != nil {
-		c.JSON(http.StatusBadGateway, gin.H{"error": err.Error()})
-		return
-	}
-	if err := engine.StartService(ctx, service.ID.String()); err != nil {
+	if err := serviceLifecycle(ctx, db, engine, service.ID, "restart"); err != nil {
 		c.JSON(http.StatusBadGateway, gin.H{"error": err.Error()})
 		return
 	}