feat(deploy): webhook queue+nocache, restart sweep, secrets at rest

- Webhook-triggered deploys now go through the per-service deploy queue
  and default to no_cache builds (dflow convention — prevents stale
  layer surprises on push).
- Boot sweep marks deployments interrupted by restart as failed instead
  of leaving phantom queued/building rows.
- is_secret environment variables are AES-GCM-encrypted at rest
  (enc:v1: prefix); key from SECRETS_KEY or JWT_SECRET fallback.
  Existing plaintext secrets encrypt on boot; deploy/exec paths decrypt
  transparently. Masked-echo writes preserve stored ciphertext.

Generated with [Devin](https://devin.ai)

Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com>

app/backend/internal/api/git_receiver.go

@@ -14,6 +14,7 @@ import (
 
 	"containr/internal/database"
 	"containr/internal/deployment"
+	"containr/internal/deployqueue"
 
 	"github.com/gin-gonic/gin"
 	"github.com/google/uuid"
@@ -178,11 +179,24 @@ func handleGitWebhookPush(c *gin.Context) {
 			`UPDATE services SET status = 'building', updated_at = $1 WHERE id = $2`,
 			time.Now(), service.ID,
 		)
-		go runDeploymentAndSync(
-			context.Background(), db, engine, &d, service,
-			CreateDeploymentRequest{CommitHash: commit, Branch: branch, Trigger: "webhook"},
-			repoUserID,
-		)
+		// Webhook pushes default to a clean build — stale layers are the
+		// classic silent-rollback failure (dflow convention).
+		webhookReq := CreateDeploymentRequest{CommitHash: commit, Branch: branch, Trigger: "webhook", NoCache: true}
+		if pos := getDeployQueue(c).Enqueue(service.ID, deployqueue.Job{
+			DeploymentID: d.ID,
+			Run: func(jctx context.Context) {
+				runDeploymentAndSync(jctx, db, engine, &d, service, webhookReq, repoUserID)
+			},
+		}); pos > 0 {
+			_, _ = db.Exec(
+				`UPDATE deployments SET status = 'queued', updated_at = $1 WHERE id = $2`,
+				time.Now(), d.ID,
+			)
+			_, _ = db.Exec(
+				`UPDATE services SET status = 'queued', updated_at = $1 WHERE id = $2`,
+				time.Now(), service.ID,
+			)
+		}
 		enqueued++
 	}
 

app/backend/internal/api/routes.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"log"
 	"net/http"
+	"os"
 	"time"
 
 	"containr/internal/build"
@@ -16,6 +17,7 @@ import (
 	"containr/internal/metrics"
 	"containr/internal/middleware"
 	"containr/internal/scaling"
+	"containr/internal/secrets"
 
 	"github.com/gin-gonic/gin"
 )
@@ -103,6 +105,41 @@ func SetupRoutes(router *gin.Engine, db *database.DB, redis *database.Redis, cfg
 	// service, the rest wait as 'queued'.
 	deployQueue := deployqueue.New()
 
+	// Encrypt-at-rest for secret env vars. SECRETS_KEY wins, JWT_SECRET is
+	// the fallback so existing installs need no new config.
+	secrets.Configure(os.Getenv("SECRETS_KEY"), cfg.JWTSecret)
+
+	// Boot sweep: in-memory engine state dies on restart, so anything that
+	// looked in-flight is actually orphaned — mark it failed rather than
+	// leaving phantom 'building'/'queued' rows.
+	_, _ = db.Exec(
+		`UPDATE deployments SET status = 'failed', error = 'Interrupted by server restart', completed_at = NOW(), updated_at = NOW()
+		 WHERE status IN ('queued', 'pending', 'building', 'deploying', 'rolling_back')`,
+	)
+
+	// Boot sweep: encrypt any plaintext secret values left over from before
+	// encryption-at-rest landed. Transparent and idempotent.
+	if encRows, err := db.Query(`SELECT id, value FROM environment_variables WHERE is_secret = TRUE`); err == nil {
+		type encRow struct {
+			id    string
+			value string
+		}
+		var toEncrypt []encRow
+		for encRows.Next() {
+			var r encRow
+			if err := encRows.Scan(&r.id, &r.value); err == nil && !secrets.IsEncrypted(r.value) {
+				toEncrypt = append(toEncrypt, r)
+			}
+		}
+		encRows.Close()
+		for _, r := range toEncrypt {
+			_, _ = db.Exec(`UPDATE environment_variables SET value = $1, updated_at = NOW() WHERE id = $2`, secrets.Encrypt(r.value), r.id)
+		}
+		if len(toEncrypt) > 0 {
+			log.Printf("secrets: encrypted %d plaintext secret variable(s)", len(toEncrypt))
+		}
+	}
+
 	// Initialize security handler
 	securityHandler := NewSecurityHandler(db, cfg.JWTSecret)
 

app/backend/internal/api/runtime.go

@@ -12,6 +12,7 @@ import (
 	"containr/internal/database"
 	"containr/internal/deployment"
 	"containr/internal/deployqueue"
+	"containr/internal/secrets"
 
 	"github.com/gin-gonic/gin"
 	"github.com/google/uuid"
@@ -79,7 +80,7 @@ func resolveServiceEnv(db *database.DB, service Service) (map[string]string, err
 		if err := rows.Scan(&k, &v); err != nil {
 			return nil, err
 		}
-		env[k] = v
+		env[k] = secrets.Decrypt(v)
 	}
 
 	var refServices []struct{ ID, Name string }
@@ -127,7 +128,7 @@ func resolveServiceEnv(db *database.DB, service Service) (map[string]string, err
 		if err != nil {
 			return "", false
 		}
-		return value, true
+		return secrets.Decrypt(value), true
 	}
 
 	for k, v := range env {

app/backend/internal/api/variables.go

@@ -2,6 +2,7 @@ package api
 
 import (
 	"containr/internal/database"
+	"containr/internal/secrets"
 	"net/http"
 	"time"
 
@@ -160,6 +161,9 @@ func handleUpdateVariables(c *gin.Context) {
 				value = stored
 			}
 		}
+		if v.IsSecret {
+			value = secrets.Encrypt(value)
+		}
 		varID := uuid.New()
 		_, err = tx.Exec(
 			`INSERT INTO environment_variables (id, service_id, key, value, is_secret, created_at, updated_at)

app/backend/internal/secrets/secrets.go

@@ -0,0 +1,98 @@
+package secrets
+
+import (
+	"crypto/aes"
+	"crypto/cipher"
+	"crypto/rand"
+	"crypto/sha256"
+	"encoding/base64"
+	"fmt"
+	"log"
+	"strings"
+	"sync"
+)
+
+// Prefix marks AES-GCM-encrypted values in environment_variables.value.
+// Plaintext rows keep working — Decrypt returns them unchanged, so the
+// rollout is transparent encrypt-on-write / decrypt-on-read.
+const Prefix = "enc:v1:"
+
+var (
+	keyOnce sync.Once
+	key     []byte
+	enabled bool
+)
+
+// Configure derives the AES-256 key. SECRETS_KEY wins; JWT_SECRET is the
+// fallback so existing installs need no new config. With neither, encrypt
+// degrades to plaintext with a single warning.
+func Configure(secretsKey, jwtSecret string) {
+	keyOnce.Do(func() {
+		seed := secretsKey
+		if seed == "" {
+			seed = jwtSecret
+		}
+		if seed == "" {
+			log.Printf("secrets: no SECRETS_KEY or JWT_SECRET — secret variables stay plaintext")
+			return
+		}
+		sum := sha256.Sum256([]byte("containr-env-secrets:" + seed))
+		key = sum[:]
+		enabled = true
+	})
+}
+
+func IsEncrypted(value string) bool {
+	return strings.HasPrefix(value, Prefix)
+}
+
+// Encrypt returns value unchanged if disabled or already encrypted.
+func Encrypt(value string) string {
+	if !enabled || IsEncrypted(value) {
+		return value
+	}
+	block, err := aes.NewCipher(key)
+	if err != nil {
+		return value
+	}
+	gcm, err := cipher.NewGCM(block)
+	if err != nil {
+		return value
+	}
+	nonce := make([]byte, gcm.NonceSize())
+	if _, err := rand.Read(nonce); err != nil {
+		return value
+	}
+	sealed := gcm.Seal(nonce, nonce, []byte(value), nil)
+	return Prefix + base64.StdEncoding.EncodeToString(sealed)
+}
+
+// Decrypt returns plaintext for enc:v1 values, the input otherwise.
+// Corrupt ciphertext fails soft to the stored value rather than erroring
+// the deploy path — an unreadable secret is no worse than a plaintext one.
+func Decrypt(value string) string {
+	if !enabled || !IsEncrypted(value) {
+		return value
+	}
+	raw, err := base64.StdEncoding.DecodeString(strings.TrimPrefix(value, Prefix))
+	if err != nil {
+		return value
+	}
+	block, err := aes.NewCipher(key)
+	if err != nil {
+		return value
+	}
+	gcm, err := cipher.NewGCM(block)
+	if err != nil {
+		return value
+	}
+	if len(raw) < gcm.NonceSize() {
+		return value
+	}
+	plain, err := gcm.Open(nil, raw[:gcm.NonceSize()], raw[gcm.NonceSize():], nil)
+	if err != nil {
+		log.Printf("secrets: decrypt failed (%v) — check SECRETS_KEY/JWT_SECRET rotation", fmt.Errorf("%w", err))
+		return value
+	}
+	return string(plain)
+}

app/backend/internal/secrets/secrets_test.go

@@ -0,0 +1,63 @@
+package secrets
+
+import (
+	"sync"
+	"testing"
+)
+
+// resetForTest restores the package state so Configure can be re-run.
+func resetForTest() {
+	keyOnce = sync.Once{}
+	key = nil
+	enabled = false
+}
+
+func TestEncryptDecryptRoundTrip(t *testing.T) {
+	resetForTest()
+	Configure("test-secrets-key", "")
+
+	enc := Encrypt("super-secret-value")
+	if !IsEncrypted(enc) {
+		t.Fatalf("expected encrypted prefix, got %q", enc)
+	}
+	if enc == "super-secret-value" {
+		t.Fatal("value stored as plaintext")
+	}
+	if got := Decrypt(enc); got != "super-secret-value" {
+		t.Fatalf("decrypt = %q", got)
+	}
+}
+
+func TestPlaintextPassThrough(t *testing.T) {
+	resetForTest()
+	Configure("test-secrets-key", "")
+	if got := Decrypt("legacy-plaintext"); got != "legacy-plaintext" {
+		t.Fatalf("legacy plaintext changed: %q", got)
+	}
+}
+
+func TestEncryptIsIdempotent(t *testing.T) {
+	resetForTest()
+	Configure("test-secrets-key", "")
+	enc := Encrypt("v")
+	if got := Encrypt(enc); got != enc {
+		t.Fatal("double-encrypt changed the value")
+	}
+}
+
+func TestDecryptFailsSoftOnCorruption(t *testing.T) {
+	resetForTest()
+	Configure("test-secrets-key", "")
+	corrupt := Prefix + "not-valid-base64!!!"
+	if got := Decrypt(corrupt); got != corrupt {
+		t.Fatal("corrupt ciphertext should pass through unchanged")
+	}
+}
+
+func TestDisabledWithoutKey(t *testing.T) {
+	resetForTest()
+	Configure("", "")
+	if got := Encrypt("v"); got != "v" {
+		t.Fatal("encrypt should pass through when no key configured")
+	}
+}

app/frontend/src/generated/docs-snapshot.ts

@@ -58,7 +58,7 @@ export const bundledDocs: BundledDoc[] = [
     "path": "guides/OPERATIONS.md",
     "section": "guides",
     "title": "Operations",
-    "markdown": "# Operations\n\nDay-2 features for running services: scheduled jobs, ad-hoc exec, managed\ndatabases, preview environments, security scans, and high availability.\n\n## Cron jobs\n\nService detail → **Cron**. Standard 5-field cron expressions (`robfig/cron`\nsyntax), per-job timezone, enable/disable, manual trigger, and execution\nhistory with captured stdout/stderr.\n\n- The scheduler ticks every minute and runs due jobs via `docker exec`\n  (`sh -c <command>`) inside the service's container.\n- Executions are retained per-job (`retention` count) and visible in the UI.\n- Requires a running container — a stopped service's due job records a\n  failure, it does not queue.\n\n```\nGET/POST /api/v1/cron-jobs            PUT/DELETE /api/v1/cron-jobs/:id\nPOST     /api/v1/cron-jobs/:id/trigger\nGET      /api/v1/cron-jobs/:id/executions\n```\n\n## Exec console\n\nService detail → **Console**. One-off `sh -c` commands against the running\ncontainer: 30s ceiling, 64KB output cap, exit code + captured output shown\nper run. Debugging aid — not an interactive shell.\n\n```\nPOST /api/v1/services/:id/exec  {command}\n```\n\n## Managed databases\n\n**/databases** page — provision PostgreSQL/Redis/MySQL/MariaDB/MongoDB/\nClickHouse/Dragonfly as managed containers on the `containr` network.\n\n- Start/stop/restart actions; connection URL with copy.\n- **Backups**: manual snapshot now, or set a cron expression\n  (`backup_schedule`) — due backups run inside the same scheduler tick as\n  cron jobs. Archives live in the `containr-db-backups` volume and are\n  downloadable (`GET /databases/:id/backups/:bid/download`) and restorable.\n- **Bind to service**: injects `connection_url` into a service's variables\n  as a secret (`DATABASE_URL` by default). Redeploy the service afterward —\n  variables are read at deploy time.\n\n## Preview environments\n\nService detail → **Previews**. Create a preview per branch/PR with a TTL,\npromote it to production, or delete it.\n\nCurrent limitation: previews are bookkeeping records — they do not build,\ndeploy, or route a real container. The generated `*.preview.containr.local`\nURL is a placeholder. Real preview runtime is tracked on the roadmap.\n\n## Security scans\n\n**/security** page — pick a project, run a scan (dependency / configuration\n/ comprehensive, optionally scoped to one service).\n\nThe scanner is heuristic config analysis, not Trivy/Grype: it flags\nunpinned image tags, `http://` source/public URLs, `npm install` without\n`npm ci`, `pip install` without hashes, `curl|sh` build steps, missing\nhealth checks, and similar smells. Findings get severity badges and can be\nresolved or ignored; the page also shows a security score, open/resolved\ncounts, scan history, and compliance status.\n\n```\nPOST /api/v1/security/scans     {project_id, service_id?, scan_type}\nGET  /api/v1/projects/:id/vulnerabilities\nPUT  /api/v1/vulnerabilities/:id          {status: open|resolved|ignored}\nGET  /api/v1/projects/:id/security/metrics\nGET  /api/v1/projects/:id/security/history\n```\n\n## High availability\n\n**/ha** page — manager enable/disable, node/health-check/alert status\ncards, manual failover (with confirmation), per-service failover policies\n(project → service picker, strategy, min healthy nodes, max failures),\nactive alerts with resolve, and health check results.\n\nHA state is in-memory like autoscaling — policies and alerts reset on\nbackend restart.\n\n## Node agents\n\n**/usage** page → *Connect node*: issue a per-node token (`cagt_…`, shown\nonce), copy the prefilled install command, revoke tokens when done. Agents\nregister over `POST /api/agents/register` and heartbeat via\n`POST /api/agents/heartbeat`.\n\n## Personal access tokens\n\n**Settings → Personal Access Tokens**: issue `cnp_…` bearer tokens for the\nCLI, MCP server, and scripts — the same privileges as your account, bounded\nby scope. The raw token is shown once at creation; only its SHA-256 hash is\nstored.\n\nScopes:\n\n- `read` — `GET`/`HEAD`/`OPTIONS` only; mutations return `403 SCOPE_INSUFFICIENT`.\n- `write` — everything except admin routes.\n- `admin` — full access; requires a platform admin account, and the token is\n  still capped at the owner's privileges.\n\nA token can never outrank its owner: non-admin scopes report\n`is_admin = false` even on admin accounts.\n\n```\nGET    /api/v1/user/tokens            list (metadata only, never hashes)\nPOST   /api/v1/user/tokens            {name, scope?, expires_in_days?} → raw token once\nDELETE /api/v1/user/tokens/:id        revoke (owner-scoped)\n```\n\nUse as `Authorization: Bearer cnp_…`, or `containr auth login cnp_…`.\nRevocation takes effect on the next request; `last_used_at` updates on each\nauthenticated call.\n"
+    "markdown": "# Operations\n\nDay-2 features for running services: scheduled jobs, ad-hoc exec, managed\ndatabases, preview environments, security scans, and high availability.\n\n## Cron jobs\n\nService detail → **Cron**. Standard 5-field cron expressions (`robfig/cron`\nsyntax), per-job timezone, enable/disable, manual trigger, and execution\nhistory with captured stdout/stderr.\n\n- The scheduler ticks every minute and runs due jobs via `docker exec`\n  (`sh -c <command>`) inside the service's container.\n- Executions are retained per-job (`retention` count) and visible in the UI.\n- Requires a running container — a stopped service's due job records a\n  failure, it does not queue.\n\n```\nGET/POST /api/v1/cron-jobs            PUT/DELETE /api/v1/cron-jobs/:id\nPOST     /api/v1/cron-jobs/:id/trigger\nGET      /api/v1/cron-jobs/:id/executions\n```\n\n## Exec console\n\nService detail → **Console**. One-off `sh -c` commands against the running\ncontainer: 30s ceiling, 64KB output cap, exit code + captured output shown\nper run. Debugging aid — not an interactive shell.\n\n```\nPOST /api/v1/services/:id/exec  {command}\n```\n\n## Managed databases\n\n**/databases** page — provision PostgreSQL/Redis/MySQL/MariaDB/MongoDB/\nClickHouse/Dragonfly as managed containers on the `containr` network.\n\n- Start/stop/restart actions; connection URL with copy.\n- **Backups**: manual snapshot now, or set a cron expression\n  (`backup_schedule`) — due backups run inside the same scheduler tick as\n  cron jobs. Archives live in the `containr-db-backups` volume and are\n  downloadable (`GET /databases/:id/backups/:bid/download`) and restorable.\n- **Bind to service**: injects `connection_url` into a service's variables\n  as a secret (`DATABASE_URL` by default). Redeploy the service afterward —\n  variables are read at deploy time.\n\n## Preview environments\n\nService detail → **Previews**. Create a preview per branch/PR with a TTL,\npromote it to production, or delete it.\n\nCurrent limitation: previews are bookkeeping records — they do not build,\ndeploy, or route a real container. The generated `*.preview.containr.local`\nURL is a placeholder. Real preview runtime is tracked on the roadmap.\n\n## Security scans\n\n**/security** page — pick a project, run a scan (dependency / configuration\n/ comprehensive, optionally scoped to one service).\n\nThe scanner is heuristic config analysis, not Trivy/Grype: it flags\nunpinned image tags, `http://` source/public URLs, `npm install` without\n`npm ci`, `pip install` without hashes, `curl|sh` build steps, missing\nhealth checks, and similar smells. Findings get severity badges and can be\nresolved or ignored; the page also shows a security score, open/resolved\ncounts, scan history, and compliance status.\n\n```\nPOST /api/v1/security/scans     {project_id, service_id?, scan_type}\nGET  /api/v1/projects/:id/vulnerabilities\nPUT  /api/v1/vulnerabilities/:id          {status: open|resolved|ignored}\nGET  /api/v1/projects/:id/security/metrics\nGET  /api/v1/projects/:id/security/history\n```\n\n## High availability\n\n**/ha** page — manager enable/disable, node/health-check/alert status\ncards, manual failover (with confirmation), per-service failover policies\n(project → service picker, strategy, min healthy nodes, max failures),\nactive alerts with resolve, and health check results.\n\nHA state is in-memory like autoscaling — policies and alerts reset on\nbackend restart.\n\n## Node agents\n\n**/usage** page → *Connect node*: issue a per-node token (`cagt_…`, shown\nonce), copy the prefilled install command, revoke tokens when done. Agents\nregister over `POST /api/agents/register` and heartbeat via\n`POST /api/agents/heartbeat`.\n\n## Personal access tokens\n\n**Settings → Personal Access Tokens**: issue `cnp_…` bearer tokens for the\nCLI, MCP server, and scripts — the same privileges as your account, bounded\nby scope. The raw token is shown once at creation; only its SHA-256 hash is\nstored.\n\nScopes:\n\n- `read` — `GET`/`HEAD`/`OPTIONS` only; mutations return `403 SCOPE_INSUFFICIENT`.\n- `write` — everything except admin routes.\n- `admin` — full access; requires a platform admin account, and the token is\n  still capped at the owner's privileges.\n\nA token can never outrank its owner: non-admin scopes report\n`is_admin = false` even on admin accounts.\n\n```\nGET    /api/v1/user/tokens            list (metadata only, never hashes)\nPOST   /api/v1/user/tokens            {name, scope?, expires_in_days?} → raw token once\nDELETE /api/v1/user/tokens/:id        revoke (owner-scoped)\n```\n\nUse as `Authorization: Bearer cnp_…`, or `containr auth login cnp_…`.\nRevocation takes effect on the next request; `last_used_at` updates on each\nauthenticated call.\n\n## Deployment queue\n\nDeployments are serialized per service: a deploy while another is active\nlands in `queued` state and promotes when it finishes — deploys, rollbacksLine truncated
   },
   {
     "path": "guides/README_BACKEND.md",

docs/guides/OPERATIONS.md

@@ -117,3 +117,20 @@ DELETE /api/v1/user/tokens/:id        revoke (owner-scoped)
 Use as `Authorization: Bearer cnp_…`, or `containr auth login cnp_…`.
 Revocation takes effect on the next request; `last_used_at` updates on each
 authenticated call.
+
+## Deployment queue
+
+Deployments are serialized per service: a deploy while another is active
+lands in `queued` state and promotes when it finishes — deploys, rollbacks,
+redeploys, and variable-apply reconciles can never race on the same service.
+`POST /deployments/:id/cancel` drops queued work or aborts running builds.
+Rows left in-flight by a server restart are marked `failed` on boot.
+Webhook-triggered deploys build with `no_cache` by default.
+
+## Secret variables
+
+Variables with `is_secret` are AES-GCM-encrypted at rest (`enc:v1:` prefix).
+Key derivation: `SECRETS_KEY` env var if set, else `JWT_SECRET`. Existing
+plaintext secrets are encrypted on boot; reads for deploy/exec decrypt
+transparently. Rotating the key makes stored secrets unreadable — re-save
+them after rotation.