feat(deploy): webhook queue+nocache, restart sweep, secrets at rest

- Webhook-triggered deploys now go through the per-service deploy queue
  and default to no_cache builds (dflow convention — prevents stale
  layer surprises on push).
- Boot sweep marks deployments interrupted by restart as failed instead
  of leaving phantom queued/building rows.
- is_secret environment variables are AES-GCM-encrypted at rest
  (enc:v1: prefix); key from SECRETS_KEY or JWT_SECRET fallback.
  Existing plaintext secrets encrypt on boot; deploy/exec paths decrypt
  transparently. Masked-echo writes preserve stored ciphertext.

Generated with [Devin](https://devin.ai)

Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com>

app/backend/internal/api/git_receiver.go

@@ -14,6 +14,7 @@ import (
 
 	"containr/internal/database"
 	"containr/internal/deployment"
+	"containr/internal/deployqueue"
 
 	"github.com/gin-gonic/gin"
 	"github.com/google/uuid"
@@ -178,11 +179,24 @@ func handleGitWebhookPush(c *gin.Context) {
 			`UPDATE services SET status = 'building', updated_at = $1 WHERE id = $2`,
 			time.Now(), service.ID,
 		)
-		go runDeploymentAndSync(
-			context.Background(), db, engine, &d, service,
-			CreateDeploymentRequest{CommitHash: commit, Branch: branch, Trigger: "webhook"},
-			repoUserID,
-		)
+		// Webhook pushes default to a clean build — stale layers are the
+		// classic silent-rollback failure (dflow convention).
+		webhookReq := CreateDeploymentRequest{CommitHash: commit, Branch: branch, Trigger: "webhook", NoCache: true}
+		if pos := getDeployQueue(c).Enqueue(service.ID, deployqueue.Job{
+			DeploymentID: d.ID,
+			Run: func(jctx context.Context) {
+				runDeploymentAndSync(jctx, db, engine, &d, service, webhookReq, repoUserID)
+			},
+		}); pos > 0 {
+			_, _ = db.Exec(
+				`UPDATE deployments SET status = 'queued', updated_at = $1 WHERE id = $2`,
+				time.Now(), d.ID,
+			)
+			_, _ = db.Exec(
+				`UPDATE services SET status = 'queued', updated_at = $1 WHERE id = $2`,
+				time.Now(), service.ID,
+			)
+		}
 		enqueued++
 	}
 

app/backend/internal/api/routes.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"log"
 	"net/http"
+	"os"
 	"time"
 
 	"containr/internal/build"
@@ -16,6 +17,7 @@ import (
 	"containr/internal/metrics"
 	"containr/internal/middleware"
 	"containr/internal/scaling"
+	"containr/internal/secrets"
 
 	"github.com/gin-gonic/gin"
 )
@@ -103,6 +105,41 @@ func SetupRoutes(router *gin.Engine, db *database.DB, redis *database.Redis, cfg
 	// service, the rest wait as 'queued'.
 	deployQueue := deployqueue.New()
 
+	// Encrypt-at-rest for secret env vars. SECRETS_KEY wins, JWT_SECRET is
+	// the fallback so existing installs need no new config.
+	secrets.Configure(os.Getenv("SECRETS_KEY"), cfg.JWTSecret)
+
+	// Boot sweep: in-memory engine state dies on restart, so anything that
+	// looked in-flight is actually orphaned — mark it failed rather than
+	// leaving phantom 'building'/'queued' rows.
+	_, _ = db.Exec(
+		`UPDATE deployments SET status = 'failed', error = 'Interrupted by server restart', completed_at = NOW(), updated_at = NOW()
+		 WHERE status IN ('queued', 'pending', 'building', 'deploying', 'rolling_back')`,
+	)
+
+	// Boot sweep: encrypt any plaintext secret values left over from before
+	// encryption-at-rest landed. Transparent and idempotent.
+	if encRows, err := db.Query(`SELECT id, value FROM environment_variables WHERE is_secret = TRUE`); err == nil {
+		type encRow struct {
+			id    string
+			value string
+		}
+		var toEncrypt []encRow
+		for encRows.Next() {
+			var r encRow
+			if err := encRows.Scan(&r.id, &r.value); err == nil && !secrets.IsEncrypted(r.value) {
+				toEncrypt = append(toEncrypt, r)
+			}
+		}
+		encRows.Close()
+		for _, r := range toEncrypt {
+			_, _ = db.Exec(`UPDATE environment_variables SET value = $1, updated_at = NOW() WHERE id = $2`, secrets.Encrypt(r.value), r.id)
+		}
+		if len(toEncrypt) > 0 {
+			log.Printf("secrets: encrypted %d plaintext secret variable(s)", len(toEncrypt))
+		}
+	}
+
 	// Initialize security handler
 	securityHandler := NewSecurityHandler(db, cfg.JWTSecret)
 

app/backend/internal/api/runtime.go

@@ -12,6 +12,7 @@ import (
 	"containr/internal/database"
 	"containr/internal/deployment"
 	"containr/internal/deployqueue"
+	"containr/internal/secrets"
 
 	"github.com/gin-gonic/gin"
 	"github.com/google/uuid"
@@ -79,7 +80,7 @@ func resolveServiceEnv(db *database.DB, service Service) (map[string]string, err
 		if err := rows.Scan(&k, &v); err != nil {
 			return nil, err
 		}
-		env[k] = v
+		env[k] = secrets.Decrypt(v)
 	}
 
 	var refServices []struct{ ID, Name string }
@@ -127,7 +128,7 @@ func resolveServiceEnv(db *database.DB, service Service) (map[string]string, err
 		if err != nil {
 			return "", false
 		}
-		return value, true
+		return secrets.Decrypt(value), true
 	}
 
 	for k, v := range env {

app/backend/internal/api/variables.go

@@ -2,6 +2,7 @@ package api
 
 import (
 	"containr/internal/database"
+	"containr/internal/secrets"
 	"net/http"
 	"time"
 
@@ -160,6 +161,9 @@ func handleUpdateVariables(c *gin.Context) {
 				value = stored
 			}
 		}
+		if v.IsSecret {
+			value = secrets.Encrypt(value)
+		}
 		varID := uuid.New()
 		_, err = tx.Exec(
 			`INSERT INTO environment_variables (id, service_id, key, value, is_secret, created_at, updated_at)

app/backend/internal/secrets/secrets.go

@@ -0,0 +1,98 @@
+package secrets
+
+import (
+	"crypto/aes"
+	"crypto/cipher"
+	"crypto/rand"
+	"crypto/sha256"
+	"encoding/base64"
+	"fmt"
+	"log"
+	"strings"
+	"sync"
+)
+
+// Prefix marks AES-GCM-encrypted values in environment_variables.value.
+// Plaintext rows keep working — Decrypt returns them unchanged, so the
+// rollout is transparent encrypt-on-write / decrypt-on-read.
+const Prefix = "enc:v1:"
+
+var (
+	keyOnce sync.Once
+	key     []byte
+	enabled bool
+)
+
+// Configure derives the AES-256 key. SECRETS_KEY wins; JWT_SECRET is the
+// fallback so existing installs need no new config. With neither, encrypt
+// degrades to plaintext with a single warning.
+func Configure(secretsKey, jwtSecret string) {
+	keyOnce.Do(func() {
+		seed := secretsKey
+		if seed == "" {
+			seed = jwtSecret
+		}
+		if seed == "" {
+			log.Printf("secrets: no SECRETS_KEY or JWT_SECRET — secret variables stay plaintext")
+			return
+		}
+		sum := sha256.Sum256([]byte("containr-env-secrets:" + seed))
+		key = sum[:]
+		enabled = true
+	})
+}
+
+func IsEncrypted(value string) bool {
+	return strings.HasPrefix(value, Prefix)
+}
+
+// Encrypt returns value unchanged if disabled or already encrypted.
+func Encrypt(value string) string {
+	if !enabled || IsEncrypted(value) {
+		return value
+	}
+	block, err := aes.NewCipher(key)
+	if err != nil {
+		return value
+	}
+	gcm, err := cipher.NewGCM(block)
+	if err != nil {
+		return value
+	}
+	nonce := make([]byte, gcm.NonceSize())
+	if _, err := rand.Read(nonce); err != nil {
+		return value
+	}
+	sealed := gcm.Seal(nonce, nonce, []byte(value), nil)
+	return Prefix + base64.StdEncoding.EncodeToString(sealed)
+}
+
+// Decrypt returns plaintext for enc:v1 values, the input otherwise.
+// Corrupt ciphertext fails soft to the stored value rather than erroring
+// the deploy path — an unreadable secret is no worse than a plaintext one.
+func Decrypt(value string) string {
+	if !enabled || !IsEncrypted(value) {
+		return value
+	}
+	raw, err := base64.StdEncoding.DecodeString(strings.TrimPrefix(value, Prefix))
+	if err != nil {
+		return value
+	}
+	block, err := aes.NewCipher(key)
+	if err != nil {
+		return value
+	}
+	gcm, err := cipher.NewGCM(block)
+	if err != nil {
+		return value
+	}
+	if len(raw) < gcm.NonceSize() {
+		return value
+	}
+	plain, err := gcm.Open(nil, raw[:gcm.NonceSize()], raw[gcm.NonceSize():], nil)
+	if err != nil {
+		log.Printf("secrets: decrypt failed (%v) — check SECRETS_KEY/JWT_SECRET rotation", fmt.Errorf("%w", err))
+		return value
+	}
+	return string(plain)
+}

app/backend/internal/secrets/secrets_test.go

@@ -0,0 +1,63 @@
+package secrets
+
+import (
+	"sync"
+	"testing"
+)
+
+// resetForTest restores the package state so Configure can be re-run.
+func resetForTest() {
+	keyOnce = sync.Once{}
+	key = nil
+	enabled = false
+}
+
+func TestEncryptDecryptRoundTrip(t *testing.T) {
+	resetForTest()
+	Configure("test-secrets-key", "")
+
+	enc := Encrypt("super-secret-value")
+	if !IsEncrypted(enc) {
+		t.Fatalf("expected encrypted prefix, got %q", enc)
+	}
+	if enc == "super-secret-value" {
+		t.Fatal("value stored as plaintext")
+	}
+	if got := Decrypt(enc); got != "super-secret-value" {
+		t.Fatalf("decrypt = %q", got)
+	}
+}
+
+func TestPlaintextPassThrough(t *testing.T) {
+	resetForTest()
+	Configure("test-secrets-key", "")
+	if got := Decrypt("legacy-plaintext"); got != "legacy-plaintext" {
+		t.Fatalf("legacy plaintext changed: %q", got)
+	}
+}
+
+func TestEncryptIsIdempotent(t *testing.T) {
+	resetForTest()
+	Configure("test-secrets-key", "")
+	enc := Encrypt("v")
+	if got := Encrypt(enc); got != enc {
+		t.Fatal("double-encrypt changed the value")
+	}
+}
+
+func TestDecryptFailsSoftOnCorruption(t *testing.T) {
+	resetForTest()
+	Configure("test-secrets-key", "")
+	corrupt := Prefix + "not-valid-base64!!!"
+	if got := Decrypt(corrupt); got != corrupt {
+		t.Fatal("corrupt ciphertext should pass through unchanged")
+	}
+}
+
+func TestDisabledWithoutKey(t *testing.T) {
+	resetForTest()
+	Configure("", "")
+	if got := Encrypt("v"); got != "v" {
+		t.Fatal("encrypt should pass through when no key configured")
+	}
+}