From 747b714a02fc1ada3e5c5ef04768f437eb5d5067 Mon Sep 17 00:00:00 2001 From: Tomas Dvorak Date: Thu, 8 Oct 2026 16:27:10 +0200 Subject: [PATCH] feat(deploy): webhook queue+nocache, restart sweep, secrets at rest MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Webhook-triggered deploys now go through the per-service deploy queue and default to no_cache builds (dflow convention — prevents stale layer surprises on push). - Boot sweep marks deployments interrupted by restart as failed instead of leaving phantom queued/building rows. - is_secret environment variables are AES-GCM-encrypted at rest (enc:v1: prefix); key from SECRETS_KEY or JWT_SECRET fallback. Existing plaintext secrets encrypt on boot; deploy/exec paths decrypt transparently. Masked-echo writes preserve stored ciphertext. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- app/backend/internal/api/git_receiver.go | 24 ++++- app/backend/internal/api/routes.go | 37 ++++++++ app/backend/internal/api/runtime.go | 5 +- app/backend/internal/api/variables.go | 4 + app/backend/internal/secrets/secrets.go | 98 ++++++++++++++++++++ app/backend/internal/secrets/secrets_test.go | 63 +++++++++++++ app/frontend/src/generated/docs-snapshot.ts | 2 +- docs/guides/OPERATIONS.md | 17 ++++ 8 files changed, 242 insertions(+), 8 deletions(-) create mode 100644 app/backend/internal/secrets/secrets.go create mode 100644 app/backend/internal/secrets/secrets_test.go diff --git a/app/backend/internal/api/git_receiver.go b/app/backend/internal/api/git_receiver.go index 299beac..a019f28 100644 --- a/app/backend/internal/api/git_receiver.go +++ b/app/backend/internal/api/git_receiver.go @@ -14,6 +14,7 @@ import ( "containr/internal/database" "containr/internal/deployment" + "containr/internal/deployqueue" "github.com/gin-gonic/gin" "github.com/google/uuid" @@ -178,11 +179,24 @@ func handleGitWebhookPush(c *gin.Context) { `UPDATE services SET status = 'building', updated_at = $1 WHERE id = $2`, time.Now(), service.ID, ) - go runDeploymentAndSync( - context.Background(), db, engine, &d, service, - CreateDeploymentRequest{CommitHash: commit, Branch: branch, Trigger: "webhook"}, - repoUserID, - ) + // Webhook pushes default to a clean build — stale layers are the + // classic silent-rollback failure (dflow convention). + webhookReq := CreateDeploymentRequest{CommitHash: commit, Branch: branch, Trigger: "webhook", NoCache: true} + if pos := getDeployQueue(c).Enqueue(service.ID, deployqueue.Job{ + DeploymentID: d.ID, + Run: func(jctx context.Context) { + runDeploymentAndSync(jctx, db, engine, &d, service, webhookReq, repoUserID) + }, + }); pos > 0 { + _, _ = db.Exec( + `UPDATE deployments SET status = 'queued', updated_at = $1 WHERE id = $2`, + time.Now(), d.ID, + ) + _, _ = db.Exec( + `UPDATE services SET status = 'queued', updated_at = $1 WHERE id = $2`, + time.Now(), service.ID, + ) + } enqueued++ } diff --git a/app/backend/internal/api/routes.go b/app/backend/internal/api/routes.go index 960cd89..4b463b7 100644 --- a/app/backend/internal/api/routes.go +++ b/app/backend/internal/api/routes.go @@ -4,6 +4,7 @@ import ( "context" "log" "net/http" + "os" "time" "containr/internal/build" @@ -16,6 +17,7 @@ import ( "containr/internal/metrics" "containr/internal/middleware" "containr/internal/scaling" + "containr/internal/secrets" "github.com/gin-gonic/gin" ) @@ -103,6 +105,41 @@ func SetupRoutes(router *gin.Engine, db *database.DB, redis *database.Redis, cfg // service, the rest wait as 'queued'. deployQueue := deployqueue.New() + // Encrypt-at-rest for secret env vars. SECRETS_KEY wins, JWT_SECRET is + // the fallback so existing installs need no new config. + secrets.Configure(os.Getenv("SECRETS_KEY"), cfg.JWTSecret) + + // Boot sweep: in-memory engine state dies on restart, so anything that + // looked in-flight is actually orphaned — mark it failed rather than + // leaving phantom 'building'/'queued' rows. + _, _ = db.Exec( + `UPDATE deployments SET status = 'failed', error = 'Interrupted by server restart', completed_at = NOW(), updated_at = NOW() + WHERE status IN ('queued', 'pending', 'building', 'deploying', 'rolling_back')`, + ) + + // Boot sweep: encrypt any plaintext secret values left over from before + // encryption-at-rest landed. Transparent and idempotent. + if encRows, err := db.Query(`SELECT id, value FROM environment_variables WHERE is_secret = TRUE`); err == nil { + type encRow struct { + id string + value string + } + var toEncrypt []encRow + for encRows.Next() { + var r encRow + if err := encRows.Scan(&r.id, &r.value); err == nil && !secrets.IsEncrypted(r.value) { + toEncrypt = append(toEncrypt, r) + } + } + encRows.Close() + for _, r := range toEncrypt { + _, _ = db.Exec(`UPDATE environment_variables SET value = $1, updated_at = NOW() WHERE id = $2`, secrets.Encrypt(r.value), r.id) + } + if len(toEncrypt) > 0 { + log.Printf("secrets: encrypted %d plaintext secret variable(s)", len(toEncrypt)) + } + } + // Initialize security handler securityHandler := NewSecurityHandler(db, cfg.JWTSecret) diff --git a/app/backend/internal/api/runtime.go b/app/backend/internal/api/runtime.go index dee02cf..781f891 100644 --- a/app/backend/internal/api/runtime.go +++ b/app/backend/internal/api/runtime.go @@ -12,6 +12,7 @@ import ( "containr/internal/database" "containr/internal/deployment" "containr/internal/deployqueue" + "containr/internal/secrets" "github.com/gin-gonic/gin" "github.com/google/uuid" @@ -79,7 +80,7 @@ func resolveServiceEnv(db *database.DB, service Service) (map[string]string, err if err := rows.Scan(&k, &v); err != nil { return nil, err } - env[k] = v + env[k] = secrets.Decrypt(v) } var refServices []struct{ ID, Name string } @@ -127,7 +128,7 @@ func resolveServiceEnv(db *database.DB, service Service) (map[string]string, err if err != nil { return "", false } - return value, true + return secrets.Decrypt(value), true } for k, v := range env { diff --git a/app/backend/internal/api/variables.go b/app/backend/internal/api/variables.go index 6982b0f..5e045af 100644 --- a/app/backend/internal/api/variables.go +++ b/app/backend/internal/api/variables.go @@ -2,6 +2,7 @@ package api import ( "containr/internal/database" + "containr/internal/secrets" "net/http" "time" @@ -160,6 +161,9 @@ func handleUpdateVariables(c *gin.Context) { value = stored } } + if v.IsSecret { + value = secrets.Encrypt(value) + } varID := uuid.New() _, err = tx.Exec( `INSERT INTO environment_variables (id, service_id, key, value, is_secret, created_at, updated_at) diff --git a/app/backend/internal/secrets/secrets.go b/app/backend/internal/secrets/secrets.go new file mode 100644 index 0000000..5c03657 --- /dev/null +++ b/app/backend/internal/secrets/secrets.go @@ -0,0 +1,98 @@ +package secrets + +import ( + "crypto/aes" + "crypto/cipher" + "crypto/rand" + "crypto/sha256" + "encoding/base64" + "fmt" + "log" + "strings" + "sync" +) + +// Prefix marks AES-GCM-encrypted values in environment_variables.value. +// Plaintext rows keep working — Decrypt returns them unchanged, so the +// rollout is transparent encrypt-on-write / decrypt-on-read. +const Prefix = "enc:v1:" + +var ( + keyOnce sync.Once + key []byte + enabled bool +) + +// Configure derives the AES-256 key. SECRETS_KEY wins; JWT_SECRET is the +// fallback so existing installs need no new config. With neither, encrypt +// degrades to plaintext with a single warning. +func Configure(secretsKey, jwtSecret string) { + keyOnce.Do(func() { + seed := secretsKey + if seed == "" { + seed = jwtSecret + } + if seed == "" { + log.Printf("secrets: no SECRETS_KEY or JWT_SECRET — secret variables stay plaintext") + return + } + sum := sha256.Sum256([]byte("containr-env-secrets:" + seed)) + key = sum[:] + enabled = true + }) +} + +func IsEncrypted(value string) bool { + return strings.HasPrefix(value, Prefix) +} + +// Encrypt returns value unchanged if disabled or already encrypted. +func Encrypt(value string) string { + if !enabled || IsEncrypted(value) { + return value + } + block, err := aes.NewCipher(key) + if err != nil { + return value + } + gcm, err := cipher.NewGCM(block) + if err != nil { + return value + } + nonce := make([]byte, gcm.NonceSize()) + if _, err := rand.Read(nonce); err != nil { + return value + } + sealed := gcm.Seal(nonce, nonce, []byte(value), nil) + return Prefix + base64.StdEncoding.EncodeToString(sealed) +} + +// Decrypt returns plaintext for enc:v1 values, the input otherwise. +// Corrupt ciphertext fails soft to the stored value rather than erroring +// the deploy path — an unreadable secret is no worse than a plaintext one. +func Decrypt(value string) string { + if !enabled || !IsEncrypted(value) { + return value + } + raw, err := base64.StdEncoding.DecodeString(strings.TrimPrefix(value, Prefix)) + if err != nil { + return value + } + block, err := aes.NewCipher(key) + if err != nil { + return value + } + gcm, err := cipher.NewGCM(block) + if err != nil { + return value + } + if len(raw) < gcm.NonceSize() { + return value + } + plain, err := gcm.Open(nil, raw[:gcm.NonceSize()], raw[gcm.NonceSize():], nil) + if err != nil { + log.Printf("secrets: decrypt failed (%v) — check SECRETS_KEY/JWT_SECRET rotation", fmt.Errorf("%w", err)) + return value + } + return string(plain) +} diff --git a/app/backend/internal/secrets/secrets_test.go b/app/backend/internal/secrets/secrets_test.go new file mode 100644 index 0000000..b5d493c --- /dev/null +++ b/app/backend/internal/secrets/secrets_test.go @@ -0,0 +1,63 @@ +package secrets + +import ( + "sync" + "testing" +) + +// resetForTest restores the package state so Configure can be re-run. +func resetForTest() { + keyOnce = sync.Once{} + key = nil + enabled = false +} + +func TestEncryptDecryptRoundTrip(t *testing.T) { + resetForTest() + Configure("test-secrets-key", "") + + enc := Encrypt("super-secret-value") + if !IsEncrypted(enc) { + t.Fatalf("expected encrypted prefix, got %q", enc) + } + if enc == "super-secret-value" { + t.Fatal("value stored as plaintext") + } + if got := Decrypt(enc); got != "super-secret-value" { + t.Fatalf("decrypt = %q", got) + } +} + +func TestPlaintextPassThrough(t *testing.T) { + resetForTest() + Configure("test-secrets-key", "") + if got := Decrypt("legacy-plaintext"); got != "legacy-plaintext" { + t.Fatalf("legacy plaintext changed: %q", got) + } +} + +func TestEncryptIsIdempotent(t *testing.T) { + resetForTest() + Configure("test-secrets-key", "") + enc := Encrypt("v") + if got := Encrypt(enc); got != enc { + t.Fatal("double-encrypt changed the value") + } +} + +func TestDecryptFailsSoftOnCorruption(t *testing.T) { + resetForTest() + Configure("test-secrets-key", "") + corrupt := Prefix + "not-valid-base64!!!" + if got := Decrypt(corrupt); got != corrupt { + t.Fatal("corrupt ciphertext should pass through unchanged") + } +} + +func TestDisabledWithoutKey(t *testing.T) { + resetForTest() + Configure("", "") + if got := Encrypt("v"); got != "v" { + t.Fatal("encrypt should pass through when no key configured") + } +} diff --git a/app/frontend/src/generated/docs-snapshot.ts b/app/frontend/src/generated/docs-snapshot.ts index 69c9b99..bfe42ba 100644 --- a/app/frontend/src/generated/docs-snapshot.ts +++ b/app/frontend/src/generated/docs-snapshot.ts @@ -58,7 +58,7 @@ export const bundledDocs: BundledDoc[] = [ "path": "guides/OPERATIONS.md", "section": "guides", "title": "Operations", - "markdown": "# Operations\n\nDay-2 features for running services: scheduled jobs, ad-hoc exec, managed\ndatabases, preview environments, security scans, and high availability.\n\n## Cron jobs\n\nService detail → **Cron**. Standard 5-field cron expressions (`robfig/cron`\nsyntax), per-job timezone, enable/disable, manual trigger, and execution\nhistory with captured stdout/stderr.\n\n- The scheduler ticks every minute and runs due jobs via `docker exec`\n (`sh -c `) inside the service's container.\n- Executions are retained per-job (`retention` count) and visible in the UI.\n- Requires a running container — a stopped service's due job records a\n failure, it does not queue.\n\n```\nGET/POST /api/v1/cron-jobs PUT/DELETE /api/v1/cron-jobs/:id\nPOST /api/v1/cron-jobs/:id/trigger\nGET /api/v1/cron-jobs/:id/executions\n```\n\n## Exec console\n\nService detail → **Console**. One-off `sh -c` commands against the running\ncontainer: 30s ceiling, 64KB output cap, exit code + captured output shown\nper run. Debugging aid — not an interactive shell.\n\n```\nPOST /api/v1/services/:id/exec {command}\n```\n\n## Managed databases\n\n**/databases** page — provision PostgreSQL/Redis/MySQL/MariaDB/MongoDB/\nClickHouse/Dragonfly as managed containers on the `containr` network.\n\n- Start/stop/restart actions; connection URL with copy.\n- **Backups**: manual snapshot now, or set a cron expression\n (`backup_schedule`) — due backups run inside the same scheduler tick as\n cron jobs. Archives live in the `containr-db-backups` volume and are\n downloadable (`GET /databases/:id/backups/:bid/download`) and restorable.\n- **Bind to service**: injects `connection_url` into a service's variables\n as a secret (`DATABASE_URL` by default). Redeploy the service afterward —\n variables are read at deploy time.\n\n## Preview environments\n\nService detail → **Previews**. Create a preview per branch/PR with a TTL,\npromote it to production, or delete it.\n\nCurrent limitation: previews are bookkeeping records — they do not build,\ndeploy, or route a real container. The generated `*.preview.containr.local`\nURL is a placeholder. Real preview runtime is tracked on the roadmap.\n\n## Security scans\n\n**/security** page — pick a project, run a scan (dependency / configuration\n/ comprehensive, optionally scoped to one service).\n\nThe scanner is heuristic config analysis, not Trivy/Grype: it flags\nunpinned image tags, `http://` source/public URLs, `npm install` without\n`npm ci`, `pip install` without hashes, `curl|sh` build steps, missing\nhealth checks, and similar smells. Findings get severity badges and can be\nresolved or ignored; the page also shows a security score, open/resolved\ncounts, scan history, and compliance status.\n\n```\nPOST /api/v1/security/scans {project_id, service_id?, scan_type}\nGET /api/v1/projects/:id/vulnerabilities\nPUT /api/v1/vulnerabilities/:id {status: open|resolved|ignored}\nGET /api/v1/projects/:id/security/metrics\nGET /api/v1/projects/:id/security/history\n```\n\n## High availability\n\n**/ha** page — manager enable/disable, node/health-check/alert status\ncards, manual failover (with confirmation), per-service failover policies\n(project → service picker, strategy, min healthy nodes, max failures),\nactive alerts with resolve, and health check results.\n\nHA state is in-memory like autoscaling — policies and alerts reset on\nbackend restart.\n\n## Node agents\n\n**/usage** page → *Connect node*: issue a per-node token (`cagt_…`, shown\nonce), copy the prefilled install command, revoke tokens when done. Agents\nregister over `POST /api/agents/register` and heartbeat via\n`POST /api/agents/heartbeat`.\n\n## Personal access tokens\n\n**Settings → Personal Access Tokens**: issue `cnp_…` bearer tokens for the\nCLI, MCP server, and scripts — the same privileges as your account, bounded\nby scope. The raw token is shown once at creation; only its SHA-256 hash is\nstored.\n\nScopes:\n\n- `read` — `GET`/`HEAD`/`OPTIONS` only; mutations return `403 SCOPE_INSUFFICIENT`.\n- `write` — everything except admin routes.\n- `admin` — full access; requires a platform admin account, and the token is\n still capped at the owner's privileges.\n\nA token can never outrank its owner: non-admin scopes report\n`is_admin = false` even on admin accounts.\n\n```\nGET /api/v1/user/tokens list (metadata only, never hashes)\nPOST /api/v1/user/tokens {name, scope?, expires_in_days?} → raw token once\nDELETE /api/v1/user/tokens/:id revoke (owner-scoped)\n```\n\nUse as `Authorization: Bearer cnp_…`, or `containr auth login cnp_…`.\nRevocation takes effect on the next request; `last_used_at` updates on each\nauthenticated call.\n" + "markdown": "# Operations\n\nDay-2 features for running services: scheduled jobs, ad-hoc exec, managed\ndatabases, preview environments, security scans, and high availability.\n\n## Cron jobs\n\nService detail → **Cron**. Standard 5-field cron expressions (`robfig/cron`\nsyntax), per-job timezone, enable/disable, manual trigger, and execution\nhistory with captured stdout/stderr.\n\n- The scheduler ticks every minute and runs due jobs via `docker exec`\n (`sh -c `) inside the service's container.\n- Executions are retained per-job (`retention` count) and visible in the UI.\n- Requires a running container — a stopped service's due job records a\n failure, it does not queue.\n\n```\nGET/POST /api/v1/cron-jobs PUT/DELETE /api/v1/cron-jobs/:id\nPOST /api/v1/cron-jobs/:id/trigger\nGET /api/v1/cron-jobs/:id/executions\n```\n\n## Exec console\n\nService detail → **Console**. One-off `sh -c` commands against the running\ncontainer: 30s ceiling, 64KB output cap, exit code + captured output shown\nper run. Debugging aid — not an interactive shell.\n\n```\nPOST /api/v1/services/:id/exec {command}\n```\n\n## Managed databases\n\n**/databases** page — provision PostgreSQL/Redis/MySQL/MariaDB/MongoDB/\nClickHouse/Dragonfly as managed containers on the `containr` network.\n\n- Start/stop/restart actions; connection URL with copy.\n- **Backups**: manual snapshot now, or set a cron expression\n (`backup_schedule`) — due backups run inside the same scheduler tick as\n cron jobs. Archives live in the `containr-db-backups` volume and are\n downloadable (`GET /databases/:id/backups/:bid/download`) and restorable.\n- **Bind to service**: injects `connection_url` into a service's variables\n as a secret (`DATABASE_URL` by default). Redeploy the service afterward —\n variables are read at deploy time.\n\n## Preview environments\n\nService detail → **Previews**. Create a preview per branch/PR with a TTL,\npromote it to production, or delete it.\n\nCurrent limitation: previews are bookkeeping records — they do not build,\ndeploy, or route a real container. The generated `*.preview.containr.local`\nURL is a placeholder. Real preview runtime is tracked on the roadmap.\n\n## Security scans\n\n**/security** page — pick a project, run a scan (dependency / configuration\n/ comprehensive, optionally scoped to one service).\n\nThe scanner is heuristic config analysis, not Trivy/Grype: it flags\nunpinned image tags, `http://` source/public URLs, `npm install` without\n`npm ci`, `pip install` without hashes, `curl|sh` build steps, missing\nhealth checks, and similar smells. Findings get severity badges and can be\nresolved or ignored; the page also shows a security score, open/resolved\ncounts, scan history, and compliance status.\n\n```\nPOST /api/v1/security/scans {project_id, service_id?, scan_type}\nGET /api/v1/projects/:id/vulnerabilities\nPUT /api/v1/vulnerabilities/:id {status: open|resolved|ignored}\nGET /api/v1/projects/:id/security/metrics\nGET /api/v1/projects/:id/security/history\n```\n\n## High availability\n\n**/ha** page — manager enable/disable, node/health-check/alert status\ncards, manual failover (with confirmation), per-service failover policies\n(project → service picker, strategy, min healthy nodes, max failures),\nactive alerts with resolve, and health check results.\n\nHA state is in-memory like autoscaling — policies and alerts reset on\nbackend restart.\n\n## Node agents\n\n**/usage** page → *Connect node*: issue a per-node token (`cagt_…`, shown\nonce), copy the prefilled install command, revoke tokens when done. Agents\nregister over `POST /api/agents/register` and heartbeat via\n`POST /api/agents/heartbeat`.\n\n## Personal access tokens\n\n**Settings → Personal Access Tokens**: issue `cnp_…` bearer tokens for the\nCLI, MCP server, and scripts — the same privileges as your account, bounded\nby scope. The raw token is shown once at creation; only its SHA-256 hash is\nstored.\n\nScopes:\n\n- `read` — `GET`/`HEAD`/`OPTIONS` only; mutations return `403 SCOPE_INSUFFICIENT`.\n- `write` — everything except admin routes.\n- `admin` — full access; requires a platform admin account, and the token is\n still capped at the owner's privileges.\n\nA token can never outrank its owner: non-admin scopes report\n`is_admin = false` even on admin accounts.\n\n```\nGET /api/v1/user/tokens list (metadata only, never hashes)\nPOST /api/v1/user/tokens {name, scope?, expires_in_days?} → raw token once\nDELETE /api/v1/user/tokens/:id revoke (owner-scoped)\n```\n\nUse as `Authorization: Bearer cnp_…`, or `containr auth login cnp_…`.\nRevocation takes effect on the next request; `last_used_at` updates on each\nauthenticated call.\n\n## Deployment queue\n\nDeployments are serialized per service: a deploy while another is active\nlands in `queued` state and promotes when it finishes — deploys, rollbacks,\nredeploys, and variable-apply reconciles can never race on the same service.\n`POST /deployments/:id/cancel` drops queued work or aborts running builds.\nRows left in-flight by a server restart are marked `failed` on boot.\nWebhook-triggered deploys build with `no_cache` by default.\n\n## Secret variables\n\nVariables with `is_secret` are AES-GCM-encrypted at rest (`enc:v1:` prefix).\nKey derivation: `SECRETS_KEY` env var if set, else `JWT_SECRET`. Existing\nplaintext secrets are encrypted on boot; reads for deploy/exec decrypt\ntransparently. Rotating the key makes stored secrets unreadable — re-save\nthem after rotation.\n" }, { "path": "guides/README_BACKEND.md", diff --git a/docs/guides/OPERATIONS.md b/docs/guides/OPERATIONS.md index 011a913..3923fe8 100644 --- a/docs/guides/OPERATIONS.md +++ b/docs/guides/OPERATIONS.md @@ -117,3 +117,20 @@ DELETE /api/v1/user/tokens/:id revoke (owner-scoped) Use as `Authorization: Bearer cnp_…`, or `containr auth login cnp_…`. Revocation takes effect on the next request; `last_used_at` updates on each authenticated call. + +## Deployment queue + +Deployments are serialized per service: a deploy while another is active +lands in `queued` state and promotes when it finishes — deploys, rollbacks, +redeploys, and variable-apply reconciles can never race on the same service. +`POST /deployments/:id/cancel` drops queued work or aborts running builds. +Rows left in-flight by a server restart are marked `failed` on boot. +Webhook-triggered deploys build with `no_cache` by default. + +## Secret variables + +Variables with `is_secret` are AES-GCM-encrypted at rest (`enc:v1:` prefix). +Key derivation: `SECRETS_KEY` env var if set, else `JWT_SECRET`. Existing +plaintext secrets are encrypted on boot; reads for deploy/exec decrypt +transparently. Rotating the key makes stored secrets unreadable — re-save +them after rotation.