feat(nodes): remote ingress via Traefik file provider

Remote replicas publish an ephemeral host port; the agent reports the
assigned binding back in the create_container result and the backend
renders a per-service file-provider config into TRAEFIK_DYNAMIC_DIR
pointing Traefik at the node's mesh IP + port. Routes are rewritten on
every remote reconcile and removed on teardown or local placement.

Also: RetireService retires remote leftovers when a service returns to
local placement; remove_container is idempotent agent-side; tombstoned
replicas skip remove dispatch; the domains-stay-local ceiling is gone.

Generated with [Devin](https://devin.ai)

Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com>

app/backend/internal/api/ingress.go

@@ -0,0 +1,171 @@
+package api
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+
+	"containr/internal/database"
+	"containr/internal/database/sqlcdb"
+	"containr/internal/deployment"
+)
+
+// Remote ingress: replicas on node agents aren't on the hub's
+// containr-network, so Traefik's Docker provider can't see them. Instead
+// the backend renders one file-provider config per service into
+// TRAEFIK_DYNAMIC_DIR (a directory Traefik watches), pointing the
+// loadbalancer at the node's reachable IP + the ephemeral host port the
+// agent reported back after `create_container`.
+//
+// Empty dir → feature off (bare-metal dev). Files are best-effort: a
+// failed write never fails a reconcile.
+
+func ingressDir() string { return os.Getenv("TRAEFIK_DYNAMIC_DIR") }
+
+func ingressFile(serviceID string) string {
+	return filepath.Join(ingressDir(), "containr-"+serviceID+".yml")
+}
+
+// jsonInt coerces JSON-ish numbers (int/int32/int64/float64) to int —
+// spec fields arrive as int32, agent payloads unmarshal as float64.
+func jsonInt(v interface{}) int {
+	switch n := v.(type) {
+	case int:
+		return n
+	case int32:
+		return int(n)
+	case int64:
+		return int(n)
+	case float64:
+		return int(n)
+	}
+	return 0
+}
+
+// reportedHostPorts parses the agent's create_container result —
+// {"name": ..., "host_ports": {"80/tcp": "32768"}}. Older agents return a
+// bare name string; JSON parse failure means no port info.
+func reportedHostPorts(result string) map[string]string {
+	var out struct {
+		HostPorts map[string]string `json:"host_ports"`
+	}
+	if json.Unmarshal([]byte(result), &out) != nil {
+		return nil
+	}
+	return out.HostPorts
+}
+
+// nodeIngressIP picks the address the hub can reach the node on. Mesh
+// overlays first — the agent operator joined a mesh precisely so the hub
+// can reach it — then the reported primary IP as a LAN fallback.
+func nodeIngressIP(agent sqlcdb.NodeAgent) string {
+	var meta struct {
+		Mesh map[string]string `json:"mesh"`
+	}
+	_ = json.Unmarshal(agent.Metadata.RawMessage, &meta)
+	for _, prefix := range []string{"tailscale", "netbird", "wg", "zt"} {
+		var keys []string
+		for name := range meta.Mesh {
+			keys = append(keys, name)
+		}
+		sort.Strings(keys)
+		for _, name := range keys {
+			if strings.HasPrefix(name, prefix) && meta.Mesh[name] != "" {
+				return meta.Mesh[name]
+			}
+		}
+	}
+	return agent.IpAddress
+}
+
+// syncRemoteIngress rewrites the service's file-provider config from the
+// live remote replica set. Called after every remote reconcile so replica
+// adds/removals and node retargets converge the upstream list.
+func syncRemoteIngress(ctx context.Context, db *database.DB, spec deployment.RuntimeSpec) {
+	if ingressDir() == "" {
+		return
+	}
+	domains := spec.Domains
+	if len(domains) == 0 && spec.Domain != "" {
+		domains = []string{spec.Domain}
+	}
+	if len(domains) == 0 || spec.Port <= 0 {
+		removeServiceIngress(spec.ServiceID)
+		return
+	}
+
+	q := sqlcdb.New(db.DB)
+	rows, err := q.ListServiceContainers(ctx, spec.ServiceID)
+	if err != nil {
+		return
+	}
+	type upstream struct{ ip, port string }
+	var servers []upstream
+	for _, row := range rows {
+		var st struct {
+			State string `json:"state"`
+		}
+		_ = json.Unmarshal(row.Status.RawMessage, &st)
+		if st.State != "running" {
+			continue
+		}
+		var port string
+		var plist []map[string]interface{}
+		if err := json.Unmarshal(row.Ports.RawMessage, &plist); err == nil {
+			for _, p := range plist {
+				if hp, ok := p["host_port"].(float64); ok && hp > 0 {
+					port = fmt.Sprintf("%d", int(hp))
+				}
+			}
+		}
+		if port == "" {
+			continue
+		}
+		agent, err := q.GetAgent(ctx, row.NodeAgentID)
+		if err != nil {
+			continue
+		}
+		if ip := nodeIngressIP(agent); ip != "" {
+			servers = append(servers, upstream{ip, port})
+		}
+	}
+	if len(servers) == 0 {
+		removeServiceIngress(spec.ServiceID)
+		return
+	}
+
+	router := "svc-" + spec.ServiceID[:8]
+	rules := make([]string, 0, len(domains))
+	for _, d := range domains {
+		rules = append(rules, "Host(`"+d+"`)")
+	}
+	var b strings.Builder
+	b.WriteString("http:\n  routers:\n    " + router + ":\n")
+	b.WriteString("      rule: \"" + strings.Join(rules, " || ") + "\"\n")
+	b.WriteString("      entrypoints: [web]\n")
+	b.WriteString("      service: " + router + "\n")
+	b.WriteString("  services:\n    " + router + ":\n      loadBalancer:\n        servers:\n")
+	for _, s := range servers {
+		b.WriteString("          - url: \"http://" + s.ip + ":" + s.port + "\"\n")
+	}
+	// Atomic-ish: write temp then rename so Traefik's watcher never reads a
+	// truncated file.
+	tmp := ingressFile(spec.ServiceID) + ".tmp"
+	if err := os.WriteFile(tmp, []byte(b.String()), 0o644); err != nil {
+		return
+	}
+	_ = os.Rename(tmp, ingressFile(spec.ServiceID))
+}
+
+// removeServiceIngress deletes the service's remote route. Called on
+// remote teardown and when a service returns to local placement.
+func removeServiceIngress(serviceID string) {
+	if ingressDir() == "" {
+		return
+	}
+	_ = os.Remove(ingressFile(serviceID))
+}

app/backend/internal/api/noderunner.go

@@ -19,8 +19,8 @@ import (
 
 // agentNodeRunner implements deployment.NodeRunner by dispatching container
 // lifecycle commands to a node agent's command queue and waiting for the
-// reported result. Remote nodes have no Traefik — domains stay local-only
-// until per-node ingress lands.
+// reported result. Domains route through the file provider —
+// syncRemoteIngress renders per-service configs from live inventory.
 type agentNodeRunner struct {
 	q  *sqlcdb.Queries
 	db *database.DB
@@ -131,10 +131,6 @@ func (r *agentNodeRunner) RemoteRuntimeState(ctx context.Context, serviceID stri
 // targetFor and retires stale replicas on any agent — including leftovers
 // from a previous pin or a different spread assignment.
 func (r *agentNodeRunner) reconcileOnAgents(ctx context.Context, spec deployment.RuntimeSpec, targetFor func(replica int) string) (*deployment.RuntimeState, error) {
-	if len(spec.Domains) > 0 || spec.Domain != "" {
-		return nil, fmt.Errorf("domains are routed by the local Traefik — remove the service's domains or run it on the local node")
-	}
-
 	replicas := spec.Replicas
 	if replicas < 1 {
 		replicas = 1
@@ -151,7 +147,7 @@ func (r *agentNodeRunner) reconcileOnAgents(ctx context.Context, spec deployment
 		if keep {
 			continue
 		}
-		if cmdErr := r.enqueueAndWait(ctx, row.NodeAgentID, "", "remove_container",
+		if _, cmdErr := r.enqueueAndWait(ctx, row.NodeAgentID, "", "remove_container",
 			map[string]interface{}{"container_name": row.Name}); cmdErr != nil {
 			return nil, fmt.Errorf("remove stale replica %s on node %s: %w", row.Name, row.NodeAgentID, cmdErr)
 		}
@@ -185,18 +181,41 @@ func (r *agentNodeRunner) reconcileOnAgents(ctx context.Context, spec deployment
 			HealthCheck:   rawJSON(map[string]interface{}{"path": spec.HealthPath}),
 		})
 
-		if err := r.enqueueAndWait(ctx, agentID, containerID, "create_container",
-			map[string]interface{}{"container": container}); err != nil {
+		result, err := r.enqueueAndWait(ctx, agentID, containerID, "create_container",
+			map[string]interface{}{"container": container})
+		if err != nil {
 			return nil, fmt.Errorf("replica %d on node %s: %w", i, agentID, err)
 		}
-		_ = r.q.UpdateContainerStatus(ctx, sqlcdb.UpdateContainerStatusParams{
-			ID:     containerID,
-			Status: rawJSON(map[string]interface{}{"state": "running", "health": "none"}),
-		})
+		if assigned := reportedHostPorts(result); len(assigned) > 0 {
+			for _, p := range ports {
+				pm, ok := p.(map[string]interface{})
+				if !ok {
+					continue
+				}
+				cport := jsonInt(pm["container_port"])
+				proto, _ := pm["protocol"].(string)
+				if hp, ok := assigned[fmt.Sprintf("%d/%s", cport, proto)]; ok {
+					if n, err := strconv.Atoi(hp); err == nil {
+						pm["host_port"] = n
+					}
+				}
+			}
+			_ = r.q.UpdateContainerPorts(ctx, sqlcdb.UpdateContainerPortsParams{
+				ID:     containerID,
+				Status: rawJSON(map[string]interface{}{"state": "running", "health": "none"}),
+				Ports:  rawJSON(ports),
+			})
+		} else {
+			_ = r.q.UpdateContainerStatus(ctx, sqlcdb.UpdateContainerStatusParams{
+				ID:     containerID,
+				Status: rawJSON(map[string]interface{}{"state": "running", "health": "none"}),
+			})
+		}
 		state.Containers = append(state.Containers, deployment.RuntimeContainer{
 			ID: containerID, Name: name, State: "running", Replica: i,
 		})
 	}
+	syncRemoteIngress(ctx, r.db, spec)
 	return state, nil
 }
 
@@ -296,7 +315,17 @@ func (r *agentNodeRunner) ControlService(ctx context.Context, serviceID, agentID
 		if row.ServiceID != serviceID {
 			continue
 		}
-		if err := r.enqueueAndWait(ctx, agentID, row.ID, cmdType,
+		// Tombstoned replicas are already gone remotely — skip the dispatch
+		// or a fresh docker rm on a missing name reports a false failure.
+		if action == "remove" {
+			var st struct {
+				State string `json:"state"`
+			}
+			if err := json.Unmarshal(row.Status.RawMessage, &st); err == nil && st.State == "removed" {
+				continue
+			}
+		}
+		if _, err := r.enqueueAndWait(ctx, agentID, row.ID, cmdType,
 			map[string]interface{}{"container_name": row.Name}); err != nil {
 			return fmt.Errorf("%s %s on node: %w", action, row.Name, err)
 		}
@@ -314,16 +343,38 @@ func (r *agentNodeRunner) RemoveService(ctx context.Context, serviceID, agentID
 	if err := r.ControlService(ctx, serviceID, agentID, "remove"); err != nil {
 		return err
 	}
-	return r.q.DeleteServiceContainersOnAgent(ctx, sqlcdb.DeleteServiceContainersOnAgentParams{
+	if err := r.q.DeleteServiceContainersOnAgent(ctx, sqlcdb.DeleteServiceContainersOnAgentParams{
 		NodeAgentID: agentID,
 		ServiceID:   serviceID,
-	})
+	}); err != nil {
+		return err
+	}
+	removeServiceIngress(serviceID)
+	return nil
+}
+
+// RetireService removes the service's remote replicas wherever they run —
+// called when the service converges to local placement so a cleared pin or
+// removed tag requirement doesn't leave containers behind.
+func (r *agentNodeRunner) RetireService(ctx context.Context, serviceID string) error {
+	agentIDs, err := r.q.ListServiceAgents(ctx, serviceID)
+	if err != nil {
+		return err
+	}
+	for _, agentID := range agentIDs {
+		if err := r.RemoveService(ctx, serviceID, agentID); err != nil {
+			return fmt.Errorf("retire on node %s: %w", agentID, err)
+		}
+	}
+	removeServiceIngress(serviceID)
+	return nil
 }
 
 // enqueueAndWait pushes a command to the agent's queue and polls for the
-// reported result. Agents poll on their own interval (≥5s), so this blocks —
-// it runs inside the deploy queue worker, never on a request path.
-func (r *agentNodeRunner) enqueueAndWait(ctx context.Context, agentID, containerID, cmdType string, payload map[string]interface{}) error {
+// reported result, returning the agent's result string on success. Agents
+// poll on their own interval (≥5s), so this blocks — it runs inside the
+// deploy queue worker, never on a request path.
+func (r *agentNodeRunner) enqueueAndWait(ctx context.Context, agentID, containerID, cmdType string, payload map[string]interface{}) (string, error) {
 	var cid sql.NullString
 	if containerID != "" {
 		cid = sql.NullString{String: containerID, Valid: true}
@@ -336,7 +387,7 @@ func (r *agentNodeRunner) enqueueAndWait(ctx context.Context, agentID, container
 		Payload:     rawJSON(payload),
 	})
 	if err != nil {
-		return err
+		return "", err
 	}
 
 	deadline := time.Now().Add(8 * time.Minute) // image pulls on a cold node
@@ -348,20 +399,20 @@ func (r *agentNodeRunner) enqueueAndWait(ctx context.Context, agentID, container
 		if err == nil {
 			switch row.Status.String {
 			case "completed":
-				return nil
+				return row.Result.String, nil
 			case "failed":
 				if row.Error.Valid && row.Error.String != "" {
-					return fmt.Errorf("agent: %s", row.Error.String)
+					return "", fmt.Errorf("agent: %s", row.Error.String)
 				}
-				return fmt.Errorf("agent reported failure")
+				return "", fmt.Errorf("agent reported failure")
 			}
 		}
 		if time.Now().After(deadline) {
-			return fmt.Errorf("command %s timed out waiting for the agent", cmdType)
+			return "", fmt.Errorf("command %s timed out waiting for the agent", cmdType)
 		}
 		select {
 		case <-ctx.Done():
-			return ctx.Err()
+			return "", ctx.Err()
 		case <-time.After(2 * time.Second):
 		}
 	}

app/backend/internal/database/sqlcdb/agents.sql.go

@@ -765,8 +765,36 @@ func (q *Queries) ListSchedulableAgentsMatching(ctx context.Context, dollar_1 js
 	return items, nil
 }
 
+const listServiceAgents = `-- name: ListServiceAgents :many
+SELECT DISTINCT node_agent_id FROM container_instances WHERE service_id = $1
+`
+
+// Distinct agents holding inventory rows for a service.
+func (q *Queries) ListServiceAgents(ctx context.Context, serviceID string) ([]string, error) {
+	rows, err := q.db.QueryContext(ctx, listServiceAgents, serviceID)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+	items := []string{}
+	for rows.Next() {
+		var node_agent_id string
+		if err := rows.Scan(&node_agent_id); err != nil {
+			return nil, err
+		}
+		items = append(items, node_agent_id)
+	}
+	if err := rows.Close(); err != nil {
+		return nil, err
+	}
+	if err := rows.Err(); err != nil {
+		return nil, err
+	}
+	return items, nil
+}
+
 const listServiceContainers = `-- name: ListServiceContainers :many
-SELECT id, name, node_agent_id, status FROM container_instances
+SELECT id, name, node_agent_id, status, ports FROM container_instances
 WHERE service_id = $1
 `
 
@@ -775,6 +803,7 @@ type ListServiceContainersRow struct {
 	Name        string                `json:"name"`
 	NodeAgentID string                `json:"node_agent_id"`
 	Status      pqtype.NullRawMessage `json:"status"`
+	Ports       pqtype.NullRawMessage `json:"ports"`
 }
 
 func (q *Queries) ListServiceContainers(ctx context.Context, serviceID string) ([]ListServiceContainersRow, error) {
@@ -791,6 +820,7 @@ func (q *Queries) ListServiceContainers(ctx context.Context, serviceID string) (
 			&i.Name,
 			&i.NodeAgentID,
 			&i.Status,
+			&i.Ports,
 		); err != nil {
 			return nil, err
 		}
@@ -1005,6 +1035,24 @@ func (q *Queries) UpdateAgentHeartbeat(ctx context.Context, arg UpdateAgentHeart
 	return err
 }
 
+const updateContainerPorts = `-- name: UpdateContainerPorts :exec
+UPDATE container_instances SET status = $2, ports = $3, updated_at = NOW()
+WHERE id = $1
+`
+
+type UpdateContainerPortsParams struct {
+	ID     string                `json:"id"`
+	Status pqtype.NullRawMessage `json:"status"`
+	Ports  pqtype.NullRawMessage `json:"ports"`
+}
+
+// Agent reports the real host bindings after create — persist them with
+// the status flip so ingress can route to the assigned port.
+func (q *Queries) UpdateContainerPorts(ctx context.Context, arg UpdateContainerPortsParams) error {
+	_, err := q.db.ExecContext(ctx, updateContainerPorts, arg.ID, arg.Status, arg.Ports)
+	return err
+}
+
 const updateContainerStatus = `-- name: UpdateContainerStatus :exec
 UPDATE container_instances SET status = $2, updated_at = NOW()
 WHERE id = $1

app/backend/internal/database/sqlcdb/querier.go

@@ -98,6 +98,8 @@ type Querier interface {
 	ListSchedulableAgents(ctx context.Context) ([]ListSchedulableAgentsRow, error)
 	// Only agents carrying every required placement tag (jsonb array containment).
 	ListSchedulableAgentsMatching(ctx context.Context, dollar_1 json.RawMessage) ([]ListSchedulableAgentsMatchingRow, error)
+	// Distinct agents holding inventory rows for a service.
+	ListServiceAgents(ctx context.Context, serviceID string) ([]string, error)
 	ListServiceContainers(ctx context.Context, serviceID string) ([]ListServiceContainersRow, error)
 	ListServiceTemplatesByCategoryForUser(ctx context.Context, arg ListServiceTemplatesByCategoryForUserParams) ([]ServiceTemplate, error)
 	ListServiceTemplatesForUser(ctx context.Context, ownerID uuid.NullUUID) ([]ServiceTemplate, error)
@@ -134,6 +136,9 @@ type Querier interface {
 	UpdateAgentHeartbeat(ctx context.Context, arg UpdateAgentHeartbeatParams) error
 	UpdateBackupTargetByIDAndUser(ctx context.Context, arg UpdateBackupTargetByIDAndUserParams) error
 	UpdateBanner(ctx context.Context, arg UpdateBannerParams) (Banner, error)
+	// Agent reports the real host bindings after create — persist them with
+	// the status flip so ingress can route to the assigned port.
+	UpdateContainerPorts(ctx context.Context, arg UpdateContainerPortsParams) error
 	UpdateContainerStatus(ctx context.Context, arg UpdateContainerStatusParams) error
 	UpdateDatabaseServiceNameAndPlanByIDAndUser(ctx context.Context, arg UpdateDatabaseServiceNameAndPlanByIDAndUserParams) error
 	UpdateDatabaseServiceNameByIDAndUser(ctx context.Context, arg UpdateDatabaseServiceNameByIDAndUserParams) error

app/backend/internal/deployment/engine.go

@@ -32,6 +32,9 @@ type NodeRunner interface {
 	// RemoteRuntimeState reads the service's remote replicas from inventory
 	// across all agents. nil/empty means nothing runs remotely.
 	RemoteRuntimeState(ctx context.Context, serviceID string) (*RuntimeState, error)
+	// RetireService removes every remote replica of the service across all
+	// agents — used when a service returns to local placement.
+	RetireService(ctx context.Context, serviceID string) error
 }
 
 type DeploymentEngine struct {

app/backend/internal/deployment/runtime.go

@@ -182,6 +182,12 @@ func (de *DeploymentEngine) ReconcileService(ctx context.Context, spec RuntimeSp
 		}
 		return de.nodeRunner.ReconcileAuto(ctx, spec)
 	}
+	// Local placement: retire any remote leftovers from a previous pin,
+	// spread, or tag affinity before reconciling local replicas.
+	if de.nodeRunner != nil {
+		_ = de.nodeRunner.RetireService(ctx, spec.ServiceID)
+	}
+
 	if spec.Replicas < 1 {
 		spec.Replicas = 1
 	}