mirror of
https://github.com/Dvorinka/Containr.git
synced 2026-10-28 07:41:27 +00:00
feat(scheduling): replica spread + resource-aware auto placement (#56)
services.spread distributes replicas across every online, schedulable agent (replica i → agents[i mod n], deterministic). auto placement now orders by real headroom — memory utilisation, cpu usage, then container count — instead of count alone. Lifecycle (start/stop/restart/delete) and GET /runtime are now driven by container_instances inventory, so remote replicas are visible and tear down wherever they run — covers spread, pins, and stale pin leftovers. Spread and explicit pins are mutually exclusive. Generated with [Devin](https://devin.ai) Co-authored-by: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
Tomáš DvořákandDevin authored and GitHub committed
1 parent
bc93c83cf9
commit
dd5ce80ce2
21 files changed
+441
-72
No files matched your search
+10
-3
@@ -289,11 +289,18 @@ and wired refs, one click or one CLI call.
|
||||
(drain evicts remote containers via the agent queue + unpins services
|
||||
so they redeploy locally), `nodes cordon|uncordon|drain` CLI +
|
||||
MCP (drain confirm-gated) + node-detail page controls; `auto` and
|
||||
explicit pins skip cordoned nodes. Remote nodes run registry-pulled
|
||||
explicit pins skip cordoned nodes. Spread placement shipped:
|
||||
`services.spread` distributes replicas deterministically across every
|
||||
online schedulable agent (replica i → agents[i mod n]), mutually
|
||||
exclusive with a pin; `auto` is now resource-aware — least memory
|
||||
utilisation, then cpu, then container count, with uninstrumented
|
||||
agents sorting last. Lifecycle and `GET /runtime` are
|
||||
inventory-driven, so remote replicas show up and tear down wherever
|
||||
they run. Remote nodes run registry-pulled
|
||||
images only — git builds, Traefik domains, and remote sleep are
|
||||
documented ceilings. Remaining: `scheduling_rules` spread/affinity
|
||||
documented ceilings. Remaining: `scheduling_rules` affinity
|
||||
policies, deployment status reported back from agent
|
||||
results, per-node resource-aware `auto` (currently count-based).
|
||||
results, spread weighting by real capacity.
|
||||
- [🟡] **Build-on-node** — private registry pulls shipped: the dispatch
|
||||
payload carries the project owner's `registries` credentials for the
|
||||
image host; the agent `docker login`s for the pull and logs out after,
|
||||
|
||||
@@ -425,9 +425,10 @@ func runDeploymentAndSyncWithImage(
|
||||
// on the local host, so remote placement only works for registry-pulled
|
||||
// images — git-sourced and rollback deploys fail fast with that message.
|
||||
nodeID := serviceNodeID(db, service.ID)
|
||||
if nodeID != "" && (service.GitRepo != "" || imageOverride != "") {
|
||||
spread := serviceSpread(db, service.ID)
|
||||
if (nodeID != "" || spread) && (service.GitRepo != "" || imageOverride != "") {
|
||||
failedAt := time.Now()
|
||||
msg := "service is pinned to a remote node but remote nodes only run registry-pulled images (git builds and rollbacks stay local — build-on-node is not yet supported)"
|
||||
msg := "service uses remote node placement but remote nodes only run registry-pulled images (git builds and rollbacks stay local — build-on-node is not yet supported)"
|
||||
_, _ = db.Exec(
|
||||
`UPDATE deployments SET status = 'failed', error = $1, completed_at = $2, updated_at = $2 WHERE id = $3`,
|
||||
msg, failedAt, dbDeployment.ID,
|
||||
@@ -462,6 +463,7 @@ func runDeploymentAndSyncWithImage(
|
||||
MaintenanceURL: maintURL,
|
||||
BasicAuthUsers: basicAuthUsers,
|
||||
NodeID: nodeID,
|
||||
Spread: spread,
|
||||
},
|
||||
Trigger: deployment.TriggerConfig{
|
||||
Type: req.Trigger,
|
||||
|
||||
@@ -48,6 +48,65 @@ func (r *agentNodeRunner) ReconcileOnNode(ctx context.Context, spec deployment.R
|
||||
if !agent.Status.Valid || agent.Status.String != "online" {
|
||||
return nil, fmt.Errorf("node %s is %s — heartbeats resume scheduling when it reconnects", agent.Name, agent.Status.String)
|
||||
}
|
||||
return r.reconcileOnAgents(ctx, spec, func(int) string { return spec.NodeID })
|
||||
}
|
||||
|
||||
// ReconcileSpread distributes replicas deterministically across online,
|
||||
// schedulable agents: replica i lands on agents[i % len(agents)], agents
|
||||
// sorted by id. Replicas wrap when they outnumber nodes. The local host is
|
||||
// not a spread target — pin "local" or leave unset for local execution.
|
||||
func (r *agentNodeRunner) ReconcileSpread(ctx context.Context, spec deployment.RuntimeSpec) (*deployment.RuntimeState, error) {
|
||||
agents, err := r.q.ListSchedulableAgents(ctx)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("list schedulable nodes: %w", err)
|
||||
}
|
||||
if len(agents) == 0 {
|
||||
return nil, fmt.Errorf("no online schedulable nodes — connect an agent or unset spread")
|
||||
}
|
||||
return r.reconcileOnAgents(ctx, spec, func(i int) string {
|
||||
return agents[i%len(agents)].ID
|
||||
})
|
||||
}
|
||||
|
||||
// RemoteRuntimeState reads the service's remote replicas from inventory.
|
||||
// Rows only carry the last dispatched state — the agent doesn't stream
|
||||
// container events — so "running" means "last create succeeded".
|
||||
func (r *agentNodeRunner) RemoteRuntimeState(ctx context.Context, serviceID string) (*deployment.RuntimeState, error) {
|
||||
rows, err := r.q.ListServiceContainers(ctx, serviceID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
state := &deployment.RuntimeState{Status: "stopped"}
|
||||
running := 0
|
||||
for _, row := range rows {
|
||||
var status struct {
|
||||
State string `json:"state"`
|
||||
}
|
||||
unmarshalRaw(row.Status, &status)
|
||||
if status.State == "" || status.State == "removed" {
|
||||
continue
|
||||
}
|
||||
replica := -1
|
||||
if idx, err := strconv.Atoi(strings.TrimPrefix(row.Name, fmt.Sprintf("containr-%s-", serviceID))); err == nil {
|
||||
replica = idx
|
||||
}
|
||||
state.Containers = append(state.Containers, deployment.RuntimeContainer{
|
||||
ID: row.ID, Name: row.Name, State: status.State, Replica: replica,
|
||||
})
|
||||
if status.State == "running" {
|
||||
running++
|
||||
}
|
||||
}
|
||||
if running > 0 {
|
||||
state.Status = "running"
|
||||
}
|
||||
return state, nil
|
||||
}
|
||||
|
||||
// reconcileOnAgents creates the service's replicas on the agents chosen by
|
||||
// targetFor and retires stale replicas on any agent — including leftovers
|
||||
// from a previous pin or a different spread assignment.
|
||||
func (r *agentNodeRunner) reconcileOnAgents(ctx context.Context, spec deployment.RuntimeSpec, targetFor func(replica int) string) (*deployment.RuntimeState, error) {
|
||||
if len(spec.Domains) > 0 || spec.Domain != "" {
|
||||
return nil, fmt.Errorf("domains are routed by the local Traefik — remove the service's domains or run it on the local node")
|
||||
}
|
||||
@@ -57,31 +116,68 @@ func (r *agentNodeRunner) ReconcileOnNode(ctx context.Context, spec deployment.R
|
||||
replicas = 1
|
||||
}
|
||||
|
||||
// Retire replicas above the desired count using inventory rows.
|
||||
prefix := fmt.Sprintf("containr-%s-", spec.ServiceID)
|
||||
existing, _ := r.q.ListContainersForAgent(ctx, spec.NodeID)
|
||||
existing, _ := r.q.ListServiceContainers(ctx, spec.ServiceID)
|
||||
for _, row := range existing {
|
||||
if row.ServiceID != spec.ServiceID || !strings.HasPrefix(row.Name, prefix) {
|
||||
if !strings.HasPrefix(row.Name, prefix) {
|
||||
continue
|
||||
}
|
||||
replica, err := strconv.Atoi(strings.TrimPrefix(row.Name, prefix))
|
||||
if err == nil && replica >= replicas {
|
||||
if cmdErr := r.enqueueAndWait(ctx, spec.NodeID, "", "remove_container",
|
||||
keep := err == nil && replica < replicas && row.NodeAgentID == targetFor(replica)
|
||||
if keep {
|
||||
continue
|
||||
}
|
||||
if cmdErr := r.enqueueAndWait(ctx, row.NodeAgentID, "", "remove_container",
|
||||
map[string]interface{}{"container_name": row.Name}); cmdErr != nil {
|
||||
return nil, fmt.Errorf("remove stale replica %s: %w", row.Name, cmdErr)
|
||||
return nil, fmt.Errorf("remove stale replica %s on node %s: %w", row.Name, row.NodeAgentID, cmdErr)
|
||||
}
|
||||
_ = r.q.UpdateContainerStatus(ctx, sqlcdb.UpdateContainerStatusParams{
|
||||
ID: row.ID,
|
||||
Status: rawJSON(map[string]interface{}{"state": "removed"}),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
state := &deployment.RuntimeState{Desired: replicas, Status: "running"}
|
||||
for i := 0; i < replicas; i++ {
|
||||
agentID := targetFor(i)
|
||||
name := remoteContainerName(spec.ServiceID, i)
|
||||
containerID := remoteContainerID(spec.NodeID, spec.ServiceID, i)
|
||||
containerID := remoteContainerID(agentID, spec.ServiceID, i)
|
||||
container, env, ports, volumes, restart := r.replicaPayload(spec, agentID, i, name)
|
||||
|
||||
_ = r.q.UpsertServiceContainer(ctx, sqlcdb.UpsertServiceContainerParams{
|
||||
ID: containerID,
|
||||
Name: name,
|
||||
Image: spec.Image,
|
||||
ProjectID: spec.ProjectID,
|
||||
ServiceID: spec.ServiceID,
|
||||
NodeAgentID: agentID,
|
||||
Status: rawJSON(map[string]interface{}{"state": "created", "health": "none"}),
|
||||
Resources: rawJSON(map[string]interface{}{"memory": spec.MemoryBytes, "cpus": float64(spec.NanoCPUs) / 1e9}),
|
||||
Ports: rawJSON(ports),
|
||||
Environment: rawJSON(env),
|
||||
Volumes: rawJSON(volumes),
|
||||
Networks: rawJSON([]string{}),
|
||||
RestartPolicy: rawJSON(map[string]interface{}{"name": restart}),
|
||||
HealthCheck: rawJSON(map[string]interface{}{"path": spec.HealthPath}),
|
||||
})
|
||||
|
||||
if err := r.enqueueAndWait(ctx, agentID, containerID, "create_container",
|
||||
map[string]interface{}{"container": container}); err != nil {
|
||||
return nil, fmt.Errorf("replica %d on node %s: %w", i, agentID, err)
|
||||
}
|
||||
_ = r.q.UpdateContainerStatus(ctx, sqlcdb.UpdateContainerStatusParams{
|
||||
ID: containerID,
|
||||
Status: rawJSON(map[string]interface{}{"state": "running", "health": "none"}),
|
||||
})
|
||||
state.Containers = append(state.Containers, deployment.RuntimeContainer{
|
||||
ID: containerID, Name: name, State: "running", Replica: i,
|
||||
})
|
||||
}
|
||||
return state, nil
|
||||
}
|
||||
|
||||
// replicaPayload builds the agent create_container payload for one replica.
|
||||
func (r *agentNodeRunner) replicaPayload(spec deployment.RuntimeSpec, agentID string, i int, name string) (map[string]interface{}, map[string]interface{}, []interface{}, []interface{}, string) {
|
||||
ports := []interface{}{}
|
||||
if spec.Port > 0 {
|
||||
hostPort := spec.PublishedPort // 0 → ephemeral on the node
|
||||
@@ -113,7 +209,7 @@ func (r *agentNodeRunner) ReconcileOnNode(ctx context.Context, spec deployment.R
|
||||
for _, arg := range spec.Command {
|
||||
cmd = append(cmd, arg)
|
||||
}
|
||||
container := map[string]interface{}{
|
||||
return map[string]interface{}{
|
||||
"name": name,
|
||||
"image": spec.Image,
|
||||
"command": cmd,
|
||||
@@ -129,40 +225,9 @@ func (r *agentNodeRunner) ReconcileOnNode(ctx context.Context, spec deployment.R
|
||||
"containr.project": spec.ProjectID,
|
||||
"containr.service": spec.ServiceID,
|
||||
"containr.replica": fmt.Sprint(i),
|
||||
"containr.node": spec.NodeID,
|
||||
"containr.node": agentID,
|
||||
},
|
||||
}
|
||||
|
||||
_ = r.q.UpsertServiceContainer(ctx, sqlcdb.UpsertServiceContainerParams{
|
||||
ID: containerID,
|
||||
Name: name,
|
||||
Image: spec.Image,
|
||||
ProjectID: spec.ProjectID,
|
||||
ServiceID: spec.ServiceID,
|
||||
NodeAgentID: spec.NodeID,
|
||||
Status: rawJSON(map[string]interface{}{"state": "created", "health": "none"}),
|
||||
Resources: rawJSON(map[string]interface{}{"memory": spec.MemoryBytes, "cpus": float64(spec.NanoCPUs) / 1e9}),
|
||||
Ports: rawJSON(ports),
|
||||
Environment: rawJSON(env),
|
||||
Volumes: rawJSON(volumes),
|
||||
Networks: rawJSON([]string{}),
|
||||
RestartPolicy: rawJSON(map[string]interface{}{"name": restart}),
|
||||
HealthCheck: rawJSON(map[string]interface{}{"path": spec.HealthPath}),
|
||||
})
|
||||
|
||||
if err := r.enqueueAndWait(ctx, spec.NodeID, containerID, "create_container",
|
||||
map[string]interface{}{"container": container}); err != nil {
|
||||
return nil, fmt.Errorf("replica %d on %s: %w", i, agent.Name, err)
|
||||
}
|
||||
_ = r.q.UpdateContainerStatus(ctx, sqlcdb.UpdateContainerStatusParams{
|
||||
ID: containerID,
|
||||
Status: rawJSON(map[string]interface{}{"state": "running", "health": "none"}),
|
||||
})
|
||||
state.Containers = append(state.Containers, deployment.RuntimeContainer{
|
||||
ID: containerID, Name: name, State: "running", Replica: i,
|
||||
})
|
||||
}
|
||||
return state, nil
|
||||
}, env, ports, volumes, restart
|
||||
}
|
||||
|
||||
// pullCredentials resolves the project owner's registry auth for the image
|
||||
|
||||
@@ -18,13 +18,37 @@ import (
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
// serviceLifecycle routes a start/stop/restart action to the node agent when
|
||||
// the service is pinned, or to the local Docker engine otherwise. Remote
|
||||
// actions block on the agent's poll loop; that matches the synchronous
|
||||
// behavior the local path already has.
|
||||
// remoteAgentsForService lists every agent holding inventory rows for the
|
||||
// service — covers pinned placements, spread replicas, and leftovers from a
|
||||
// since-cleared pin.
|
||||
func remoteAgentsForService(ctx context.Context, db *database.DB, serviceID uuid.UUID) []string {
|
||||
rows, err := db.Query(
|
||||
`SELECT DISTINCT node_agent_id FROM container_instances WHERE service_id = $1`,
|
||||
serviceID,
|
||||
)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
defer rows.Close()
|
||||
var ids []string
|
||||
for rows.Next() {
|
||||
var id string
|
||||
if rows.Scan(&id) == nil && id != "" {
|
||||
ids = append(ids, id)
|
||||
}
|
||||
}
|
||||
return ids
|
||||
}
|
||||
|
||||
// serviceLifecycle routes a start/stop/restart action to every agent holding
|
||||
// the service's replicas, then to the local Docker engine. Both run — a
|
||||
// service can't be pinned and local at once, but stale containers from a
|
||||
// cleared pin shouldn't survive a stop.
|
||||
func serviceLifecycle(ctx context.Context, db *database.DB, engine *deployment.DeploymentEngine, serviceID uuid.UUID, action string) error {
|
||||
if nodeID := serviceNodeID(db, serviceID); nodeID != "" {
|
||||
return engine.ControlServiceOnNode(ctx, serviceID.String(), nodeID, action)
|
||||
for _, agentID := range remoteAgentsForService(ctx, db, serviceID) {
|
||||
if err := engine.ControlServiceOnNode(ctx, serviceID.String(), agentID, action); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
switch action {
|
||||
case "start":
|
||||
@@ -40,12 +64,11 @@ func serviceLifecycle(ctx context.Context, db *database.DB, engine *deployment.D
|
||||
return fmt.Errorf("unsupported lifecycle action %q", action)
|
||||
}
|
||||
|
||||
// removeServiceRuntime tears down replicas wherever they run — remote node
|
||||
// when pinned, local Docker otherwise.
|
||||
// removeServiceRuntime tears down replicas wherever they run — remote nodes
|
||||
// (pinned or spread) and the local Docker host.
|
||||
func removeServiceRuntime(ctx context.Context, db *database.DB, engine *deployment.DeploymentEngine, serviceID uuid.UUID) {
|
||||
if nodeID := serviceNodeID(db, serviceID); nodeID != "" {
|
||||
_ = engine.RemoveServiceContainersOnNode(ctx, serviceID.String(), nodeID)
|
||||
return
|
||||
for _, agentID := range remoteAgentsForService(ctx, db, serviceID) {
|
||||
_ = engine.RemoveServiceContainersOnNode(ctx, serviceID.String(), agentID)
|
||||
}
|
||||
_ = engine.RemoveServiceContainers(ctx, serviceID.String())
|
||||
}
|
||||
@@ -70,6 +93,7 @@ func serviceRuntimeSpec(db *database.DB, service Service) (deployment.RuntimeSpe
|
||||
HealthPath: service.HealthCheckPath,
|
||||
RestartPolicy: service.RestartPolicy,
|
||||
NodeID: serviceNodeID(db, service.ID),
|
||||
Spread: serviceSpread(db, service.ID),
|
||||
}
|
||||
// Best effort: reuse the host port from the last live deployment so the
|
||||
// public URL survives restarts. Column exists post-migration; older DBs
|
||||
@@ -374,6 +398,7 @@ func handleGetServiceRuntime(c *gin.Context) {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
|
||||
return
|
||||
}
|
||||
mergeRemoteRuntime(c.Request.Context(), engine, service.ID.String(), state)
|
||||
engine.ProbeServiceHealth(c.Request.Context(), state, service.ProjectID.String(), service.Name, service.Port, service.HealthCheckPath)
|
||||
persistPublishedPort(db, service.ID, state.Ports)
|
||||
if host := requestHostname(c); host != "" {
|
||||
@@ -564,6 +589,37 @@ func enqueueServiceRedeploy(c *gin.Context, db *database.DB, service Service) bo
|
||||
return true
|
||||
}
|
||||
|
||||
// mergeRemoteRuntime folds inventory-backed remote replica state into a
|
||||
// local runtime view so pinned and spread services report their containers.
|
||||
// Remote rows carry deterministic UUIDs, local ones Docker IDs — no overlap.
|
||||
func mergeRemoteRuntime(ctx context.Context, engine *deployment.DeploymentEngine, serviceID string, state *deployment.RuntimeState) {
|
||||
remote, err := engine.RemoteRuntimeState(ctx, serviceID)
|
||||
if err != nil || remote == nil {
|
||||
return
|
||||
}
|
||||
running := 0
|
||||
for _, c := range state.Containers {
|
||||
if c.State == "running" {
|
||||
running++
|
||||
}
|
||||
}
|
||||
for _, rc := range remote.Containers {
|
||||
state.Containers = append(state.Containers, rc)
|
||||
if rc.State == "running" {
|
||||
running++
|
||||
}
|
||||
}
|
||||
switch {
|
||||
case len(state.Containers) == 0:
|
||||
case running == 0:
|
||||
state.Status = "stopped"
|
||||
case running < state.Desired:
|
||||
state.Status = "degraded"
|
||||
default:
|
||||
state.Status = "running"
|
||||
}
|
||||
}
|
||||
|
||||
// persistPublishedPort records the host port Docker actually bound so later
|
||||
// redeploys can reuse it. Idempotent — only writes when the value changed.
|
||||
func persistPublishedPort(db *database.DB, serviceID uuid.UUID, ports []uint16) {
|
||||
@@ -587,7 +643,11 @@ func liveServiceStatus(c *gin.Context, db *database.DB, service *Service) {
|
||||
ctx, cancel := context.WithTimeout(c.Request.Context(), 3*time.Second)
|
||||
defer cancel()
|
||||
state, err := engine.RuntimeState(ctx, service.ID.String(), service.Replicas)
|
||||
if err != nil || state == nil || len(state.Containers) == 0 {
|
||||
if err != nil || state == nil {
|
||||
return
|
||||
}
|
||||
mergeRemoteRuntime(ctx, engine, service.ID.String(), state)
|
||||
if len(state.Containers) == 0 {
|
||||
return
|
||||
}
|
||||
if state.Status != service.Status {
|
||||
|
||||
@@ -53,6 +53,9 @@ type Service struct {
|
||||
// resolved "auto" picks the least-loaded online agent at write time).
|
||||
NodeID string `json:"node_id,omitempty" db:"-"`
|
||||
NodeName string `json:"node_name,omitempty" db:"-"`
|
||||
// Spread distributes replicas across every online, schedulable agent.
|
||||
// Mutually exclusive with an explicit NodeID pin.
|
||||
Spread bool `json:"spread" db:"-"`
|
||||
CreatedAt time.Time `json:"created_at" db:"created_at"`
|
||||
UpdatedAt time.Time `json:"updated_at" db:"updated_at"`
|
||||
}
|
||||
@@ -79,10 +82,21 @@ func serviceNodeID(db *database.DB, serviceID uuid.UUID) string {
|
||||
return ""
|
||||
}
|
||||
|
||||
// loadServiceNode fills the pin + the agent's display name for responses.
|
||||
// serviceSpread reads the spread column with the same lazy pattern.
|
||||
func serviceSpread(db *database.DB, serviceID uuid.UUID) bool {
|
||||
var spread bool
|
||||
if err := db.QueryRow(`SELECT COALESCE(spread, false) FROM services WHERE id = $1`, serviceID).Scan(&spread); err == nil {
|
||||
return spread
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// loadServiceNode fills the pin + spread + the agent's display name for
|
||||
// responses.
|
||||
func loadServiceNode(db *database.DB, s *Service) {
|
||||
var name sql.NullString
|
||||
s.NodeID = serviceNodeID(db, s.ID)
|
||||
s.Spread = serviceSpread(db, s.ID)
|
||||
if s.NodeID == "" {
|
||||
return
|
||||
}
|
||||
@@ -171,6 +185,9 @@ type CreateServiceRequest struct {
|
||||
// NodeID pins placement to a node agent; "auto" picks the least-loaded
|
||||
// online agent, "local"/"" runs on the Containr host.
|
||||
NodeID string `json:"node_id"`
|
||||
// Spread distributes replicas across every online, schedulable agent.
|
||||
// Mutually exclusive with an explicit node pin.
|
||||
Spread *bool `json:"spread"`
|
||||
}
|
||||
|
||||
// UpdateServiceRequest represents a request to update a service
|
||||
@@ -202,6 +219,9 @@ type UpdateServiceRequest struct {
|
||||
SleepIdleMinutes *int `json:"sleep_idle_minutes"`
|
||||
// NodeID pins placement; pointer so "local"/"" explicitly clears the pin.
|
||||
NodeID *string `json:"node_id"`
|
||||
// Spread toggles multi-node replica distribution; pointer so false is
|
||||
// a real update, not a missing field.
|
||||
Spread *bool `json:"spread"`
|
||||
}
|
||||
|
||||
// handleGetServices retrieves all services for a project
|
||||
@@ -442,14 +462,23 @@ func handleCreateService(c *gin.Context) {
|
||||
|
||||
sourceType := inferServiceSourceType(service)
|
||||
|
||||
if req.Spread != nil {
|
||||
service.Spread = *req.Spread
|
||||
}
|
||||
if req.NodeID != "" {
|
||||
if service.Spread && req.NodeID != "auto" {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": "spread and an explicit node pin are mutually exclusive", "code": "VALIDATION"})
|
||||
return
|
||||
}
|
||||
nodeID, nodeErr := resolveNodePin(c.Request.Context(), db.(*database.DB), req.NodeID)
|
||||
if nodeErr != nil {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": nodeErr.Error(), "code": "VALIDATION"})
|
||||
return
|
||||
}
|
||||
if !service.Spread {
|
||||
service.NodeID = nodeID
|
||||
}
|
||||
}
|
||||
|
||||
// Insert service into database
|
||||
_, err = db.(*database.DB).Exec(
|
||||
@@ -459,9 +488,9 @@ func handleCreateService(c *gin.Context) {
|
||||
git_repo, git_branch, build_path, cpu, memory, replicas, port, domain,
|
||||
healthcheck_path, restart_policy, volumes,
|
||||
builder, cpu_reserve, memory_reserve, static_build_cmd, static_dir,
|
||||
node_id, created_at, updated_at)
|
||||
node_id, spread, created_at, updated_at)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26,
|
||||
$27, $28, $29, $30, $31, $32, $33, $34)`,
|
||||
$27, $28, $29, $30, $31, $32, $33, $34, $35)`,
|
||||
service.ID, service.ProjectID, service.Name, environmentID, service.Type,
|
||||
sourceType, firstNonEmpty(service.GitRepo, service.Image), service.Image,
|
||||
"", service.Command, service.Type, service.Status, service.Image, service.Command,
|
||||
@@ -470,6 +499,7 @@ func handleCreateService(c *gin.Context) {
|
||||
volumesJSON, service.Builder, service.CPUReserve, service.MemoryReserve,
|
||||
service.StaticBuildCmd, service.StaticDir,
|
||||
sql.NullString{String: service.NodeID, Valid: service.NodeID != ""},
|
||||
service.Spread,
|
||||
service.CreatedAt, service.UpdatedAt,
|
||||
)
|
||||
|
||||
@@ -772,8 +802,18 @@ func handleUpdateService(c *gin.Context) {
|
||||
volumesArg = volumesJSON
|
||||
}
|
||||
|
||||
if req.Spread != nil {
|
||||
existingService.Spread = *req.Spread
|
||||
} else {
|
||||
existingService.Spread = serviceSpread(db.(*database.DB), serviceID)
|
||||
}
|
||||
if req.NodeID != nil {
|
||||
nodeID, nodeErr := resolveNodePin(c.Request.Context(), db.(*database.DB), *req.NodeID)
|
||||
nodeValue := strings.TrimSpace(*req.NodeID)
|
||||
if existingService.Spread && nodeValue != "" && nodeValue != "auto" && nodeValue != "local" {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": "spread and an explicit node pin are mutually exclusive — set spread=false first", "code": "VALIDATION"})
|
||||
return
|
||||
}
|
||||
nodeID, nodeErr := resolveNodePin(c.Request.Context(), db.(*database.DB), nodeValue)
|
||||
if nodeErr != nil {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": nodeErr.Error(), "code": "VALIDATION"})
|
||||
return
|
||||
@@ -782,6 +822,10 @@ func handleUpdateService(c *gin.Context) {
|
||||
} else {
|
||||
existingService.NodeID = serviceNodeID(db.(*database.DB), serviceID)
|
||||
}
|
||||
if existingService.Spread && existingService.NodeID != "" {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": "service already has an explicit node pin — clear it (node_id=\"local\") before enabling spread", "code": "VALIDATION"})
|
||||
return
|
||||
}
|
||||
|
||||
existingService.UpdatedAt = time.Now()
|
||||
|
||||
@@ -794,7 +838,7 @@ func handleUpdateService(c *gin.Context) {
|
||||
restart_policy = $15, volumes = COALESCE($17::jsonb, volumes), updated_at = $16,
|
||||
builder = $19, cpu_reserve = $20, memory_reserve = $21,
|
||||
static_build_cmd = $22, static_dir = $23,
|
||||
sleep_enabled = $24, sleep_idle_minutes = $25, node_id = $26
|
||||
sleep_enabled = $24, sleep_idle_minutes = $25, node_id = $26, spread = $27
|
||||
WHERE id = $18`,
|
||||
existingService.Name, existingService.Type, existingService.Image, existingService.Command,
|
||||
existingService.Environment, existingService.GitRepo, existingService.GitBranch,
|
||||
@@ -806,6 +850,7 @@ func handleUpdateService(c *gin.Context) {
|
||||
existingService.StaticBuildCmd, existingService.StaticDir,
|
||||
existingService.SleepEnabled, existingService.SleepIdleMinutes,
|
||||
sql.NullString{String: existingService.NodeID, Valid: existingService.NodeID != ""},
|
||||
existingService.Spread,
|
||||
)
|
||||
|
||||
if err != nil {
|
||||
|
||||
@@ -114,6 +114,9 @@ var servicesCreateCmd = &cobra.Command{
|
||||
if v, _ := cmd.Flags().GetInt("replicas"); v > 0 {
|
||||
body["replicas"] = v
|
||||
}
|
||||
if v, _ := cmd.Flags().GetBool("spread"); v {
|
||||
body["spread"] = v
|
||||
}
|
||||
data, err := c.Do("POST", "/projects/"+args[0]+"/services", body)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -203,6 +206,10 @@ var servicesUpdateCmd = &cobra.Command{
|
||||
v, _ := cmd.Flags().GetBool("sleep")
|
||||
body["sleep_enabled"] = v
|
||||
}
|
||||
if cmd.Flags().Changed("spread") {
|
||||
v, _ := cmd.Flags().GetBool("spread")
|
||||
body["spread"] = v
|
||||
}
|
||||
if cmd.Flags().Changed("sleep-idle") {
|
||||
v, _ := cmd.Flags().GetInt("sleep-idle")
|
||||
body["sleep_idle_minutes"] = v
|
||||
@@ -495,6 +502,7 @@ func init() {
|
||||
f.String("static-cmd", "", "build command for the static builder")
|
||||
f.String("static-dir", "", "output dir for the static builder (default dist)")
|
||||
f.String("node", "", "pin to a node agent id, or 'auto' for least-loaded (image services only)")
|
||||
f.Bool("spread", false, "distribute replicas across all online schedulable nodes")
|
||||
|
||||
uf := servicesUpdateCmd.Flags()
|
||||
uf.String("name", "", "service name")
|
||||
@@ -520,6 +528,7 @@ func init() {
|
||||
uf.Bool("sleep", false, "enable scale-to-zero on idle (sleep mode)")
|
||||
uf.Int("sleep-idle", 0, "idle minutes before sleeping (1-1440)")
|
||||
uf.String("node", "", "pin to a node agent id, 'auto' for least-loaded, 'local' to clear")
|
||||
uf.Bool("spread", false, "spread replicas across all online schedulable nodes (--spread=false to disable)")
|
||||
|
||||
for _, a := range []string{"start", "stop", "restart", "redeploy", "sleep", "wake"} {
|
||||
verb := a
|
||||
|
||||
@@ -691,6 +691,80 @@ func (q *Queries) ListPendingCommands(ctx context.Context, nodeAgentID string) (
|
||||
return items, nil
|
||||
}
|
||||
|
||||
const listSchedulableAgents = `-- name: ListSchedulableAgents :many
|
||||
SELECT id, name FROM node_agents
|
||||
WHERE status = 'online' AND schedulable
|
||||
ORDER BY id
|
||||
`
|
||||
|
||||
type ListSchedulableAgentsRow struct {
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
}
|
||||
|
||||
func (q *Queries) ListSchedulableAgents(ctx context.Context) ([]ListSchedulableAgentsRow, error) {
|
||||
rows, err := q.db.QueryContext(ctx, listSchedulableAgents)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
items := []ListSchedulableAgentsRow{}
|
||||
for rows.Next() {
|
||||
var i ListSchedulableAgentsRow
|
||||
if err := rows.Scan(&i.ID, &i.Name); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
items = append(items, i)
|
||||
}
|
||||
if err := rows.Close(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return items, nil
|
||||
}
|
||||
|
||||
const listServiceContainers = `-- name: ListServiceContainers :many
|
||||
SELECT id, name, node_agent_id, status FROM container_instances
|
||||
WHERE service_id = $1
|
||||
`
|
||||
|
||||
type ListServiceContainersRow struct {
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
NodeAgentID string `json:"node_agent_id"`
|
||||
Status pqtype.NullRawMessage `json:"status"`
|
||||
}
|
||||
|
||||
func (q *Queries) ListServiceContainers(ctx context.Context, serviceID string) ([]ListServiceContainersRow, error) {
|
||||
rows, err := q.db.QueryContext(ctx, listServiceContainers, serviceID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
items := []ListServiceContainersRow{}
|
||||
for rows.Next() {
|
||||
var i ListServiceContainersRow
|
||||
if err := rows.Scan(
|
||||
&i.ID,
|
||||
&i.Name,
|
||||
&i.NodeAgentID,
|
||||
&i.Status,
|
||||
); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
items = append(items, i)
|
||||
}
|
||||
if err := rows.Close(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return items, nil
|
||||
}
|
||||
|
||||
const pickLeastLoadedAgent = `-- name: PickLeastLoadedAgent :one
|
||||
SELECT a.id FROM node_agents a
|
||||
LEFT JOIN LATERAL (
|
||||
@@ -699,10 +773,20 @@ LEFT JOIN LATERAL (
|
||||
ORDER BY h.timestamp DESC LIMIT 1
|
||||
) h ON true
|
||||
WHERE a.status = 'online' AND a.schedulable
|
||||
ORDER BY COALESCE(h.container_count, 0) ASC, a.created_at ASC
|
||||
ORDER BY
|
||||
COALESCE(
|
||||
(a.resources->'memory'->>'used')::double precision
|
||||
/ NULLIF((a.resources->'memory'->>'total')::double precision, 0),
|
||||
1.0
|
||||
) ASC,
|
||||
COALESCE((a.resources->'cpu'->>'usage')::double precision, 100.0) ASC,
|
||||
COALESCE(h.container_count, 0) ASC,
|
||||
a.created_at ASC
|
||||
LIMIT 1
|
||||
`
|
||||
|
||||
// Resource-aware: lowest memory utilisation first, then cpu, then container
|
||||
// count. Agents without telemetry sort last (NULL → worst score).
|
||||
func (q *Queries) PickLeastLoadedAgent(ctx context.Context) (string, error) {
|
||||
row := q.db.QueryRowContext(ctx, pickLeastLoadedAgent)
|
||||
var id string
|
||||
|
||||
@@ -276,6 +276,7 @@ type Service struct {
|
||||
StaticBuildCmd string `json:"static_build_cmd"`
|
||||
StaticDir string `json:"static_dir"`
|
||||
NodeID sql.NullString `json:"node_id"`
|
||||
Spread bool `json:"spread"`
|
||||
}
|
||||
|
||||
type ServiceDomain struct {
|
||||
|
||||
@@ -94,6 +94,8 @@ type Querier interface {
|
||||
// display, when the caller owns or is a member of it, or when the caller is a
|
||||
// platform admin. Anonymous callers pass user_id = uuid.Nil, is_admin = false.
|
||||
ListProjectsWithStatsByUser(ctx context.Context, arg ListProjectsWithStatsByUserParams) ([]ListProjectsWithStatsByUserRow, error)
|
||||
ListSchedulableAgents(ctx context.Context) ([]ListSchedulableAgentsRow, error)
|
||||
ListServiceContainers(ctx context.Context, serviceID string) ([]ListServiceContainersRow, error)
|
||||
ListServiceTemplatesByCategoryForUser(ctx context.Context, arg ListServiceTemplatesByCategoryForUserParams) ([]ServiceTemplate, error)
|
||||
ListServiceTemplatesForUser(ctx context.Context, ownerID uuid.NullUUID) ([]ServiceTemplate, error)
|
||||
ListUserInvites(ctx context.Context) ([]UserInvite, error)
|
||||
@@ -102,6 +104,8 @@ type Querier interface {
|
||||
MarkAllNotificationsReadByUser(ctx context.Context, arg MarkAllNotificationsReadByUserParams) error
|
||||
MarkNotificationReadByIDAndUser(ctx context.Context, arg MarkNotificationReadByIDAndUserParams) error
|
||||
MarkUserInviteUsed(ctx context.Context, arg MarkUserInviteUsedParams) error
|
||||
// Resource-aware: lowest memory utilisation first, then cpu, then container
|
||||
// count. Agents without telemetry sort last (NULL → worst score).
|
||||
PickLeastLoadedAgent(ctx context.Context) (string, error)
|
||||
RevokeAgentAuthToken(ctx context.Context, id uuid.UUID) (AgentAuthToken, error)
|
||||
RevokeUserToken(ctx context.Context, arg RevokeUserTokenParams) (int64, error)
|
||||
|
||||
@@ -17,11 +17,17 @@ type NodeRunner interface {
|
||||
// ReconcileOnNode brings the node's containers in line with the spec and
|
||||
// returns the resulting runtime state.
|
||||
ReconcileOnNode(ctx context.Context, spec RuntimeSpec) (*RuntimeState, error)
|
||||
// ReconcileSpread distributes the service's replicas across every online,
|
||||
// schedulable agent — replica i lands on agents[i % len].
|
||||
ReconcileSpread(ctx context.Context, spec RuntimeSpec) (*RuntimeState, error)
|
||||
// ControlService fans a lifecycle action ("start"|"stop"|"restart"|"remove")
|
||||
// out to every replica of the service on the agent.
|
||||
ControlService(ctx context.Context, serviceID, agentID, action string) error
|
||||
// RemoveService tears down every replica of the service on the agent.
|
||||
RemoveService(ctx context.Context, serviceID, agentID string) error
|
||||
// RemoteRuntimeState reads the service's remote replicas from inventory
|
||||
// across all agents. nil/empty means nothing runs remotely.
|
||||
RemoteRuntimeState(ctx context.Context, serviceID string) (*RuntimeState, error)
|
||||
}
|
||||
|
||||
type DeploymentEngine struct {
|
||||
@@ -85,6 +91,9 @@ type ServiceConfig struct {
|
||||
// host. Remote placement requires a registry-pullable image — builds stay
|
||||
// on the local host (build-on-node is not yet supported).
|
||||
NodeID string `json:"node_id,omitempty"`
|
||||
// Spread distributes replicas across every online, schedulable agent.
|
||||
// Same registry-image constraint as NodeID.
|
||||
Spread bool `json:"spread,omitempty"`
|
||||
}
|
||||
|
||||
type PortMapping struct {
|
||||
@@ -376,6 +385,7 @@ func (de *DeploymentEngine) deployService(ctx context.Context, deployment *Deplo
|
||||
MaintenanceURL: cfg.MaintenanceURL,
|
||||
BasicAuthUsers: cfg.BasicAuthUsers,
|
||||
NodeID: cfg.NodeID,
|
||||
Spread: cfg.Spread,
|
||||
}
|
||||
for _, pm := range cfg.PortMappings {
|
||||
if spec.Port == 0 {
|
||||
|
||||
@@ -57,6 +57,9 @@ type RuntimeSpec struct {
|
||||
BasicAuthUsers string // htpasswd-format user:hash pairs, comma-separated
|
||||
// NodeID pins the service to a node agent; "" runs on the local host.
|
||||
NodeID string
|
||||
// Spread distributes replicas across all online, schedulable agents
|
||||
// instead of a single node. Mutually exclusive with NodeID.
|
||||
Spread bool
|
||||
}
|
||||
|
||||
// RuntimeContainer describes one live replica.
|
||||
@@ -156,6 +159,12 @@ func (de *DeploymentEngine) ListServiceContainers(ctx context.Context, serviceID
|
||||
// otherwise only the replica delta is applied. A pinned NodeID routes the
|
||||
// whole reconcile to that agent's command queue instead of local Docker.
|
||||
func (de *DeploymentEngine) ReconcileService(ctx context.Context, spec RuntimeSpec) (*RuntimeState, error) {
|
||||
if spec.Spread && spec.NodeID == "" {
|
||||
if de.nodeRunner == nil {
|
||||
return nil, fmt.Errorf("service has spread placement but remote dispatch is unavailable")
|
||||
}
|
||||
return de.nodeRunner.ReconcileSpread(ctx, spec)
|
||||
}
|
||||
if spec.NodeID != "" {
|
||||
if de.nodeRunner == nil {
|
||||
return nil, fmt.Errorf("service is pinned to node %s but remote dispatch is unavailable", spec.NodeID)
|
||||
@@ -526,6 +535,15 @@ func (de *DeploymentEngine) RemoveServiceContainersOnNode(ctx context.Context, s
|
||||
return de.nodeRunner.RemoveService(ctx, serviceID, nodeID)
|
||||
}
|
||||
|
||||
// RemoteRuntimeState exposes inventory-backed remote container state; nil
|
||||
// runner means no node support was wired at boot.
|
||||
func (de *DeploymentEngine) RemoteRuntimeState(ctx context.Context, serviceID string) (*RuntimeState, error) {
|
||||
if de.nodeRunner == nil {
|
||||
return nil, nil
|
||||
}
|
||||
return de.nodeRunner.RemoteRuntimeState(ctx, serviceID)
|
||||
}
|
||||
|
||||
// RemoveProjectContainers deletes every managed container in a project.
|
||||
func (de *DeploymentEngine) RemoveProjectContainers(ctx context.Context, projectID string) error {
|
||||
containers, err := de.dockerClient.ListContainersFiltered(ctx, filters.NewArgs(
|
||||
|
||||
@@ -53,7 +53,7 @@ func init() {
|
||||
// Services
|
||||
register(spec{"containr_services_list", "List services in a project", "GET", "/projects/{project_id}/services", []string{"project_id"}, nil, false})
|
||||
register(spec{"containr_services_get", "Get a service by id", "GET", "/services/{id}", []string{"id"}, nil, false})
|
||||
register(spec{"containr_services_create", "Create a service in a project. Args: name (required), type, image, git_repo, git_branch, environment, port, domain, replicas, volumes = [{type: volume|bind, source, target, read_only}], builder = auto|railpack|nixpacks|dockerfile|static, cpu, memory, cpu_reserve, memory_reserve, static_build_cmd, static_dir, node_id (agent id | 'auto' = least-loaded online node; image services only — remote nodes can't run git builds).", "POST", "/projects/{project_id}/services", []string{"project_id"}, nil, false})
|
||||
register(spec{"containr_services_create", "Create a service in a project. Args: name (required), type, image, git_repo, git_branch, environment, port, domain, replicas, volumes = [{type: volume|bind, source, target, read_only}], builder = auto|railpack|nixpacks|dockerfile|static, cpu, memory, cpu_reserve, memory_reserve, static_build_cmd, static_dir, node_id (agent id | 'auto' = least-loaded online node; image services only — remote nodes can't run git builds), spread (bool — distribute replicas across every online schedulable node; mutually exclusive with a node pin).", "POST", "/projects/{project_id}/services", []string{"project_id"}, nil, false})
|
||||
register(spec{"containr_services_delete", "Delete a service. Requires confirm:true.", "DELETE", "/services/{id}", []string{"id"}, nil, true})
|
||||
register(spec{"containr_services_start", "Start a service", "POST", "/services/{id}/start", []string{"id"}, nil, false})
|
||||
register(spec{"containr_services_stop", "Stop a service", "POST", "/services/{id}/stop", []string{"id"}, nil, false})
|
||||
@@ -66,7 +66,7 @@ func init() {
|
||||
register(spec{"containr_services_move", "Move a service to another project the caller owns. Args: id, project_id (required).", "POST", "/services/{id}/move", []string{"id"}, []string{"project_id"}, false})
|
||||
register(spec{"containr_services_logs", "Get service runtime logs. Args: id, tail (default 100).", "GET", "/services/{id}/logs", []string{"id"}, nil, false})
|
||||
register(spec{"containr_services_exec", "Run a one-off command in the service container (30s ceiling). Args: id, command (required).", "POST", "/services/{id}/exec", []string{"id"}, []string{"command"}, false})
|
||||
register(spec{"containr_services_update", "Update a service. Args: id, name, image, command, domain, restart_policy, healthcheck_path, cpu, memory, replicas, port, volumes = [{type: volume|bind, source, target, read_only}] — volumes replaces the whole mount list; pass [] to clear. maintenance_mode (bool), basic_auth = [{username, password}] — replaces all creds; [] clears. builder = auto|railpack|nixpacks|dockerfile|static, cpu_reserve, memory_reserve, static_build_cmd, static_dir (empty string clears reserves/static fields). sleep_enabled (bool) + sleep_idle_minutes (1-1440) enable scale-to-zero on idle; a wake placeholder answers traffic while the service resumes. node_id pins placement to a node agent ('auto' = least-loaded, 'local' = run on the Containr host).", "PUT", "/services/{id}", []string{"id"}, nil, false})
|
||||
register(spec{"containr_services_update", "Update a service. Args: id, name, image, command, domain, restart_policy, healthcheck_path, cpu, memory, replicas, port, volumes = [{type: volume|bind, source, target, read_only}] — volumes replaces the whole mount list; pass [] to clear. maintenance_mode (bool), basic_auth = [{username, password}] — replaces all creds; [] clears. builder = auto|railpack|nixpacks|dockerfile|static, cpu_reserve, memory_reserve, static_build_cmd, static_dir (empty string clears reserves/static fields). sleep_enabled (bool) + sleep_idle_minutes (1-1440) enable scale-to-zero on idle; a wake placeholder answers traffic while the service resumes. node_id pins placement to a node agent ('auto' = least-loaded, 'local' = run on the Containr host). spread (bool) distributes replicas across every online schedulable node; mutually exclusive with a pin — clear node_id to 'local' first if pinned.", "PUT", "/services/{id}", []string{"id"}, nil, false})
|
||||
|
||||
// Domains
|
||||
register(spec{"containr_domains_list", "List domains attached to a service", "GET", "/services/{id}/domains", []string{"id"}, nil, false})
|
||||
|
||||
@@ -0,0 +1,5 @@
|
||||
-- +goose Up
|
||||
ALTER TABLE services ADD COLUMN IF NOT EXISTS spread BOOLEAN NOT NULL DEFAULT false;
|
||||
|
||||
-- +goose Down
|
||||
ALTER TABLE services DROP COLUMN IF EXISTS spread;
|
||||
@@ -116,6 +116,8 @@ WHERE node_agent_id = $1 AND type = $2
|
||||
ORDER BY created_at DESC LIMIT 1;
|
||||
|
||||
-- name: PickLeastLoadedAgent :one
|
||||
-- Resource-aware: lowest memory utilisation first, then cpu, then container
|
||||
-- count. Agents without telemetry sort last (NULL → worst score).
|
||||
SELECT a.id FROM node_agents a
|
||||
LEFT JOIN LATERAL (
|
||||
SELECT h.container_count FROM agent_heartbeats h
|
||||
@@ -123,9 +125,26 @@ LEFT JOIN LATERAL (
|
||||
ORDER BY h.timestamp DESC LIMIT 1
|
||||
) h ON true
|
||||
WHERE a.status = 'online' AND a.schedulable
|
||||
ORDER BY COALESCE(h.container_count, 0) ASC, a.created_at ASC
|
||||
ORDER BY
|
||||
COALESCE(
|
||||
(a.resources->'memory'->>'used')::double precision
|
||||
/ NULLIF((a.resources->'memory'->>'total')::double precision, 0),
|
||||
1.0
|
||||
) ASC,
|
||||
COALESCE((a.resources->'cpu'->>'usage')::double precision, 100.0) ASC,
|
||||
COALESCE(h.container_count, 0) ASC,
|
||||
a.created_at ASC
|
||||
LIMIT 1;
|
||||
|
||||
-- name: ListSchedulableAgents :many
|
||||
SELECT id, name FROM node_agents
|
||||
WHERE status = 'online' AND schedulable
|
||||
ORDER BY id;
|
||||
|
||||
-- name: ListServiceContainers :many
|
||||
SELECT id, name, node_agent_id, status FROM container_instances
|
||||
WHERE service_id = $1;
|
||||
|
||||
-- name: SetAgentSchedulable :exec
|
||||
UPDATE node_agents SET schedulable = $2, updated_at = NOW() WHERE id = $1;
|
||||
|
||||
|
||||
@@ -67,7 +67,8 @@ CREATE TABLE services (
|
||||
memory_reserve VARCHAR(20) NOT NULL DEFAULT '',
|
||||
static_build_cmd VARCHAR(255) NOT NULL DEFAULT '',
|
||||
static_dir VARCHAR(255) NOT NULL DEFAULT '',
|
||||
node_id VARCHAR(255)
|
||||
node_id VARCHAR(255),
|
||||
spread BOOLEAN NOT NULL DEFAULT false
|
||||
);
|
||||
|
||||
CREATE TABLE registries (
|
||||
|
||||
@@ -2404,10 +2404,12 @@ export function ServiceDetailPage() {
|
||||
</div>
|
||||
<div className="flex items-center gap-3 text-xs">
|
||||
<select
|
||||
value={nodeDraft ?? (service.nodeId || 'local')}
|
||||
value={service.spread ? '__spread__' : (nodeDraft ?? (service.nodeId || 'local'))}
|
||||
disabled={service.spread}
|
||||
onChange={(e) => setNodeDraft(e.target.value)}
|
||||
className="px-2 py-1.5 rounded-[var(--radius-md)] bg-[var(--surface-muted)] border border-[var(--border-subtle)] text-sm text-[var(--text-primary)]"
|
||||
className="px-2 py-1.5 rounded-[var(--radius-md)] bg-[var(--surface-muted)] border border-[var(--border-subtle)] text-sm text-[var(--text-primary)] disabled:opacity-50"
|
||||
>
|
||||
{service.spread && <option value="__spread__">All nodes (spread)</option>}
|
||||
<option value="local">This server (local Docker)</option>
|
||||
<option value="auto">Auto — least-loaded online node</option>
|
||||
{(nodesQuery.data ?? []).map((n) => (
|
||||
@@ -2416,6 +2418,17 @@ export function ServiceDetailPage() {
|
||||
</option>
|
||||
))}
|
||||
</select>
|
||||
<label className="flex items-center gap-1.5 text-[var(--text-secondary)]">
|
||||
<input
|
||||
type="checkbox"
|
||||
checked={service.spread ?? false}
|
||||
disabled={updateServiceMutation.isPending || (!!service.nodeId && !service.spread)}
|
||||
title={service.nodeId && !service.spread ? 'Clear the node pin first (choose "This server")' : undefined}
|
||||
onChange={(e) => updateServiceMutation.mutate({ spread: e.target.checked })}
|
||||
className="accent-[var(--accent-primary)]"
|
||||
/>
|
||||
spread replicas across nodes
|
||||
</label>
|
||||
{nodeDraft !== null && (
|
||||
<>
|
||||
<button
|
||||
|
||||
@@ -10878,6 +10878,8 @@ export interface components {
|
||||
node_id?: string;
|
||||
/** @description Display name of the pinned node agent (read-only) */
|
||||
node_name?: string;
|
||||
/** @description Replicas are distributed across every online, schedulable node agent (replica i → agents[i mod n]). Mutually exclusive with an explicit node_id pin. */
|
||||
spread?: boolean;
|
||||
/** @description Computed public URL (published port or domain) */
|
||||
public_url?: string;
|
||||
/**
|
||||
@@ -10950,6 +10952,8 @@ export interface components {
|
||||
static_dir?: string;
|
||||
/** @description Node agent id to pin to, 'auto' = least-loaded online agent; omit/empty = local host */
|
||||
node_id?: string;
|
||||
/** @description Distribute replicas across every online, schedulable node agent; mutually exclusive with an explicit node_id pin */
|
||||
spread?: boolean;
|
||||
};
|
||||
UpdateServiceRequest: {
|
||||
/** @description Service name */
|
||||
@@ -11013,6 +11017,8 @@ export interface components {
|
||||
sleep_idle_minutes?: number;
|
||||
/** @description Node agent id to pin to, 'auto' = least-loaded online agent, 'local' = run on the Containr host (clears the pin) */
|
||||
node_id?: string;
|
||||
/** @description Distribute replicas across every online, schedulable node agent; mutually exclusive with an explicit node_id pin — send 'local' first to clear a pin */
|
||||
spread?: boolean;
|
||||
};
|
||||
ServiceDomain: {
|
||||
/** Format: uuid */
|
||||
|
||||
@@ -40,7 +40,7 @@ export const bundledDocs: BundledDoc[] = [
|
||||
"path": "guides/CLI.md",
|
||||
"section": "guides",
|
||||
"title": "Containr CLI",
|
||||
"markdown": "# Containr CLI\n\nThe CLI controls every part of a Containr instance — the same API surface\nthe web UI uses. Build it with `scripts/build-cli.sh` or\n`go build ./cmd/cli` from `app/backend`.\n\n## Authentication\n\nCreate a personal access token in the web UI under\n**Settings → Personal Access Tokens** (or `containr tokens create`), then:\n\n```bash\ncontainr auth login cnp_... # verify + store\ncontainr auth login cnp_... --url https://containr.example.com\ncontainr auth login cnp_... --name production # named profile\ncontainr auth status # show active profile + identity\ncontainr auth logout\n```\n\nResolution order for every command: `--token`/`--api-url` flags >\n`CONTAINR_TOKEN`/`CONTAINR_API_URL` env > active profile >\n`http://localhost:8080/api/v1`. The active profile is `--profile` >\n`CONTAINR_PROFILE` > `current_profile` in `~/.containr.yaml`.\n\n## Output and scripting\n\n- `--json` prints the raw API response — safe to pipe and parse.\n- `--yes` / `-y` skips confirmation on destructive commands (delete,\n revoke, restore, failover). Non-interactive shells refuse destructive\n commands without it.\n- Exit codes are stable: `0` ok, `1` error, `2` auth/permission,\n `3` not found, `4` validation, `5` unreachable.\n\n## Commands\n\n```\nauth login|logout|status token auth and profiles\ntokens list|create|revoke personal access tokens\nprojects list|get|create|delete\nservices list|get|create|update|delete|start|stop|restart|redeploy\nservices clone|move duplicate config or re-parent a service\nservices domains list|add|remove|default|check multi-domain + DNS preflight\nregistries list|add|update|remove private-image pull credentials\nvolumes list|delete docker volume inventory + cleanup (admin)\ndeploy <service-id> trigger a deployment (--no-cache, --commit-hash, --branch)\ndeployments list|get|logs|rollback|cancel\nlogs <service-id> [--follow] runtime logs (SSE follow)\nexec <service-id> -- <cmd> one-off container command\nvariables list|set|unset env vars (secrets masked; --redeploy applies them)\nvariables shared list|set|unset project shared vars — referenced as ${{shared.KEY}}\ndatabases list|get|create|update|delete|action|backup|restore|download-backup|import-backup\n import-backup <id> <file.tar.gz|-> uploads an archive as a restore\n point; ships offsite too when a backup target is assigned\ndatabases list|get|create|update|delete|action|backup|restore|download-backup\n create --public exposes the port on all interfaces;\n update <id> --public/--public=false toggles it (container recreates, data persists)\ndatabases register <name> register an external database (--type --host --port --database --username --password --ssl; probed before storing)\ndatabases test-connection probe a connection — by id, or ad-hoc flags\nbackup-targets list|add|update|remove|test\n S3-compatible archive destinations; add/update probe the bucket\n first, empty --access-key/--secret-key on update keep stored keys;\n update <db-id> --backup-target assigns a target (\"\" clears it)\ncron list|get|create|delete|trigger|executions\ntemplates list|get|create|delete|deploy|plan|import-compose|deploy-graph\n deploy --name --var K=V; plan dry-resolves expressions;\n import-compose <compose.yml|-> converts a compose file into\n a v2 graph config; deploy-graph <project-id> <config.json|->\n deploys an ad-hoc service graph\nimport compose <project-id> <compose.yml|->\n deploys a docker-compose file straight into a project —\n no template row. DB images become managed databases\n attached under their service name; --var K=V overrides\n template variables\nnodes list|get|delete|tokens node agents + onboarding tokens (admin)\nnodes add <user@host> SSH bootstrap — one session installs the agent, then never needed again (--print for manual runs)\nnodes update|prune|commands rename/auto-prune, bounded docker prune, command history\nmigrate railway pull a Railway project (services, vars, domains) via GraphQL and recreate it — --dry-run prints the plan\nmigrate dflow import a dflow project (services, vars, volumes, domains) via its Payload REST API — needs --url + --api-key, --dry-run prints the plan\nscaling status|policies|services|scale\nha status|alerts|health|policies|enable|disable|failover\nsecurity scan|vulnerabilities|metrics\ngateway services|keys|anLine truncated
|
||||
"markdown": "# Containr CLI\n\nThe CLI controls every part of a Containr instance — the same API surface\nthe web UI uses. Build it with `scripts/build-cli.sh` or\n`go build ./cmd/cli` from `app/backend`.\n\n## Authentication\n\nCreate a personal access token in the web UI under\n**Settings → Personal Access Tokens** (or `containr tokens create`), then:\n\n```bash\ncontainr auth login cnp_... # verify + store\ncontainr auth login cnp_... --url https://containr.example.com\ncontainr auth login cnp_... --name production # named profile\ncontainr auth status # show active profile + identity\ncontainr auth logout\n```\n\nResolution order for every command: `--token`/`--api-url` flags >\n`CONTAINR_TOKEN`/`CONTAINR_API_URL` env > active profile >\n`http://localhost:8080/api/v1`. The active profile is `--profile` >\n`CONTAINR_PROFILE` > `current_profile` in `~/.containr.yaml`.\n\n## Output and scripting\n\n- `--json` prints the raw API response — safe to pipe and parse.\n- `--yes` / `-y` skips confirmation on destructive commands (delete,\n revoke, restore, failover). Non-interactive shells refuse destructive\n commands without it.\n- Exit codes are stable: `0` ok, `1` error, `2` auth/permission,\n `3` not found, `4` validation, `5` unreachable.\n\n## Commands\n\n```\nauth login|logout|status token auth and profiles\ntokens list|create|revoke personal access tokens\nprojects list|get|create|delete\nservices list|get|create|update|delete|start|stop|restart|redeploy\nservices clone|move duplicate config or re-parent a service\nservices domains list|add|remove|default|check multi-domain + DNS preflight\nregistries list|add|update|remove private-image pull credentials\nvolumes list|delete docker volume inventory + cleanup (admin)\ndeploy <service-id> trigger a deployment (--no-cache, --commit-hash, --branch)\ndeployments list|get|logs|rollback|cancel\nlogs <service-id> [--follow] runtime logs (SSE follow)\nexec <service-id> -- <cmd> one-off container command\nvariables list|set|unset env vars (secrets masked; --redeploy applies them)\nvariables shared list|set|unset project shared vars — referenced as ${{shared.KEY}}\ndatabases list|get|create|update|delete|action|backup|restore|download-backup|import-backup\n import-backup <id> <file.tar.gz|-> uploads an archive as a restore\n point; ships offsite too when a backup target is assigned\ndatabases list|get|create|update|delete|action|backup|restore|download-backup\n create --public exposes the port on all interfaces;\n update <id> --public/--public=false toggles it (container recreates, data persists)\ndatabases register <name> register an external database (--type --host --port --database --username --password --ssl; probed before storing)\ndatabases test-connection probe a connection — by id, or ad-hoc flags\nbackup-targets list|add|update|remove|test\n S3-compatible archive destinations; add/update probe the bucket\n first, empty --access-key/--secret-key on update keep stored keys;\n update <db-id> --backup-target assigns a target (\"\" clears it)\ncron list|get|create|delete|trigger|executions\ntemplates list|get|create|delete|deploy|plan|import-compose|deploy-graph\n deploy --name --var K=V; plan dry-resolves expressions;\n import-compose <compose.yml|-> converts a compose file into\n a v2 graph config; deploy-graph <project-id> <config.json|->\n deploys an ad-hoc service graph\nimport compose <project-id> <compose.yml|->\n deploys a docker-compose file straight into a project —\n no template row. DB images become managed databases\n attached under their service name; --var K=V overrides\n template variables\nnodes list|get|delete|tokens node agents + onboarding tokens (admin)\nnodes add <user@host> SSH bootstrap — one session installs the agent, then never needed again (--print for manual runs)\nnodes update|prune|commands rename/auto-prune, bounded docker prune, command history\nmigrate railway pull a Railway project (services, vars, domains) via GraphQL and recreate it — --dry-run prints the plan\nmigrate dflow import a dflow project (services, vars, volumes, domains) via its Payload REST API — needs --url + --api-key, --dry-run prints the plan\nscaling status|policies|services|scale\nha status|alerts|health|policies|enable|disable|failover\nsecurity scan|vulnerabilities|metrics\ngateway services|keys|anLine truncated
|
||||
},
|
||||
{
|
||||
"path": "guides/CLOUDFLARE_SETUP.md",
|
||||
|
||||
@@ -86,6 +86,7 @@ export type ServiceEntity = {
|
||||
sleepIdleMinutes?: number;
|
||||
nodeId?: string;
|
||||
nodeName?: string;
|
||||
spread?: boolean;
|
||||
publicUrl?: string;
|
||||
};
|
||||
|
||||
@@ -696,6 +697,7 @@ function normalizeService(service: RawService): ServiceEntity | null {
|
||||
sleepIdleMinutes: service.sleep_idle_minutes,
|
||||
nodeId: service.node_id,
|
||||
nodeName: service.node_name,
|
||||
spread: service.spread,
|
||||
publicUrl: service.public_url,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -6974,6 +6974,9 @@ components:
|
||||
node_name:
|
||||
type: string
|
||||
description: Display name of the pinned node agent (read-only)
|
||||
spread:
|
||||
type: boolean
|
||||
description: Replicas are distributed across every online, schedulable node agent (replica i → agents[i mod n]). Mutually exclusive with an explicit node_id pin.
|
||||
public_url:
|
||||
type: string
|
||||
description: Computed public URL (published port or domain)
|
||||
@@ -7068,6 +7071,9 @@ components:
|
||||
node_id:
|
||||
type: string
|
||||
description: Node agent id to pin to, 'auto' = least-loaded online agent; omit/empty = local host
|
||||
spread:
|
||||
type: boolean
|
||||
description: Distribute replicas across every online, schedulable node agent; mutually exclusive with an explicit node_id pin
|
||||
|
||||
UpdateServiceRequest:
|
||||
type: object
|
||||
@@ -7157,6 +7163,9 @@ components:
|
||||
node_id:
|
||||
type: string
|
||||
description: Node agent id to pin to, 'auto' = least-loaded online agent, 'local' = run on the Containr host (clears the pin)
|
||||
spread:
|
||||
type: boolean
|
||||
description: Distribute replicas across every online, schedulable node agent; mutually exclusive with an explicit node_id pin — send 'local' first to clear a pin
|
||||
|
||||
ServiceDomain:
|
||||
type: object
|
||||
|
||||
@@ -143,6 +143,15 @@ containr services update <id> --node auto # least-loaded online node
|
||||
containr services update <id> --node local # move back to this server
|
||||
```
|
||||
|
||||
`auto` scores nodes by real headroom — lowest memory utilisation, then
|
||||
cpu, then container count. Or spread replicas across every online,
|
||||
schedulable node (mutually exclusive with a pin):
|
||||
|
||||
```bash
|
||||
containr services update <id> --replicas 4 --spread # replica i → node[i % n]
|
||||
containr services update <id> --spread=false # back to single placement
|
||||
```
|
||||
|
||||
Remote nodes run registry-pulled images only — git builds, domains
|
||||
(Traefik lives on the host), and sleep detection stay local. Start,
|
||||
stop, restart, redeploy, and delete all fan out to the pinned node's
|
||||
|
||||
Reference in new issue
Block a user