feat: phase 6 production hardening + DB-to-DB transfer

Security: - Graceful shutdown: http.Server + signal.NotifyContext, 10s drain, retention sweeper cancelled before pool close - Scoped API keys (migration 00006): ingest/read/write/admin enforced in auth middleware; scope picker + badges in Settings UI; scopeless legacy keys act as admin. Read-only keys now get 403 on ingest (verified live) - nginx: baseline security headers on all responses, CSP scoped to SPA; fixed add_header inheritance shadowing - better-auth rate limit enabled unconditionally (was gated on NODE_ENV, which the shipped .env sets to development) - Mailpit moved to docker-compose.dev.yml; prod compose cannot expose it - setup.sh generates PRIMORA_ENCRYPTION_KEY Reliability/ops: - Retention: per-project retention_{events,audit,webhook}_days (migration 00007, 0=off) + hourly RetentionSweeper; edit UI in Projects - GET /api/v1/metrics Prometheus exposition - compose: healthchecks + restart policies everywhere, dragonfly pinned to v2.0.0, depends_on service_healthy - backup.sh/restore.sh (pg_dump -Fc + storage volume); restore verified against scratch postgres+volume - .dockerignore: exclude target dirs (build context was 4.7GB) Correctness/testing: - DBX transfer endpoint: POST /db-connections/:id/transfer — SQL or Redis target, 1000-row bound, server-side creds (verified live pg→dragonfly) - Pagination audit: ListComponents capped at 500 (auto-registers) - CI: goose up/reset/up job + compose e2e-smoke job (smoke-e2e.sh verified locally: signup, bootstrap, scoped keys, ingest, metrics) - k6 load sanity script: 6650 reqs, ~217/s, 0 real failures, p95 221ms (429s = configured rate limiter working) Docs: DEPLOYMENT_GUIDE.md, PRODUCTION_READINESS.md; verify script repaired (stale refs + set -e arithmetic abort). ROADMAP: phase 2 + phase 6 marked complete. Authored By: TDvorak <[email protected]> Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com>
2026-10-28 07:41:23 +00:00 · 2026-09-17 09:15:42 +02:00
1 parent 1b34dd43bb
commit 55edbf8b56
@@ -0,0 +1,50 @@
+#!/usr/bin/env bash
+# Primora backup: pg_dump of the database + archive of the object-storage root.
+#
+# Produces backups/primora-YYYYMMDD-HHMMSS.tar.gz containing:
+#   db.dump       postgres custom-format dump (all schemas, incl. core + auth)
+#   storage/      contents of the backend_storage volume
+#
+# Usage (from the repo root, against the compose stack):
+#   ./scripts/backup.sh
+#
+# Optional env:
+#   BACKUP_DIR           output directory            (default: ./backups)
+#   POSTGRES_CONTAINER   compose container name      (default: primora-postgres-1)
+#   POSTGRES_USER/DB     dump credentials            (default: primora/primora)
+#   STORAGE_VOLUME       docker volume for objects   (default: primora_backend_storage)
+#
+# Restore with scripts/restore.sh. Schedule via cron or a systemd timer —
+# see DEPLOYMENT_GUIDE.md#backups.
+
+set -euo pipefail
+
+BACKUP_DIR="${BACKUP_DIR:-./backups}"
+POSTGRES_CONTAINER="${POSTGRES_CONTAINER:-primora-postgres-1}"
+POSTGRES_USER="${POSTGRES_USER:-primora}"
+POSTGRES_DB="${POSTGRES_DB:-primora}"
+STORAGE_VOLUME="${STORAGE_VOLUME:-primora_backend_storage}"
+
+stamp="$(date +%Y%m%d-%H%M%S)"
+mkdir -p "$BACKUP_DIR"
+BACKUP_DIR="$(cd "$BACKUP_DIR" && pwd)"
+# Workdir lives under BACKUP_DIR — /tmp is not bind-mountable on every
+# docker setup, and the storage archive is written through a container.
+work="$(mktemp -d "$BACKUP_DIR/.tmp.XXXXXX")"
+out="$BACKUP_DIR/primora-$stamp.tar.gz"
+trap 'rm -rf "$work"' EXIT
+
+echo "==> dumping database from $POSTGRES_CONTAINER"
+docker exec "$POSTGRES_CONTAINER" \
+  pg_dump -U "$POSTGRES_USER" -d "$POSTGRES_DB" -Fc \
+  > "$work/db.dump"
+[ -s "$work/db.dump" ] || { echo "pg_dump produced an empty file" >&2; exit 1; }
+
+echo "==> archiving storage volume $STORAGE_VOLUME"
+# Read the volume through a throwaway container — no host mount assumptions.
+docker run --rm -v "$STORAGE_VOLUME:/data:ro" -v "$work:/backup" \
+  alpine:3 tar -C /data -cf /backup/storage.tar .
+[ -s "$work/storage.tar" ] || echo "warning: storage volume appears empty" >&2
+
+tar -C "$work" -czf "$out" db.dump storage.tar
+echo "==> wrote $out ($(du -h "$out" | cut -f1))"
@@ -0,0 +1,75 @@
+// Primora load sanity — NOT a benchmark. Verifies the ingest path and a hot
+// read endpoint hold up under modest concurrency without errors.
+//
+//   k6 run scripts/load-sanity.js \
+//     -e PRIMORA_API_KEY=prm_xxx_xxx \
+//     -e PRIMORA_PROJECT_ID=<uuid> \
+//     -e BASE_URL=http://localhost
+//
+// The key needs the ingest + read scopes. Create one in Settings → API keys.
+
+import http from "k6/http";
+import { check } from "k6";
+
+const BASE = __ENV.BASE_URL || "http://localhost";
+const API_KEY = __ENV.PRIMORA_API_KEY || "";
+const PROJECT = __ENV.PRIMORA_PROJECT_ID || "";
+
+export const options = {
+  scenarios: {
+    ingest: {
+      exec: "ingest",
+      executor: "constant-vus",
+      vus: 5,
+      duration: "30s",
+    },
+    hot_read: {
+      exec: "hotRead",
+      executor: "constant-vus",
+      vus: 10,
+      duration: "30s",
+    },
+  },
+  // 429s are the rate limiter doing its job (API_KEY_RATE_LIMIT_PER_MINUTE,
+  // default 600/min) — counted as expected via responseCallback below. The
+  // sanity bar is: no real errors, no hangs. Raise the env limit or lower
+  // VUs to exercise unthrottled throughput.
+  thresholds: {
+    http_req_failed: ["rate<0.01"],
+    http_req_duration: ["p(95)<500"],
+  },
+};
+
+const ingestBody = JSON.stringify({
+  events: [
+    {
+      type: "metric",
+      severity: "info",
+      component: "load-sanity",
+      message: "k6 sanity metric",
+      payload: { name: "k6.sanity", value: 1 },
+    },
+  ],
+});
+
+export function ingest() {
+  const res = http.post(`${BASE}/api/v1/ingest`, ingestBody, {
+    headers: {
+      "content-type": "application/json",
+      "X-Primora-Key": API_KEY,
+    },
+    responseCallback: http.expectedStatuses(202, 429),
+  });
+  check(res, { "ingest 202 or throttled": (r) => r.status === 202 || r.status === 429 });
+}
+
+export function hotRead() {
+  const res = http.get(
+    `${BASE}/api/v1/projects/${PROJECT}/events?limit=25`,
+    {
+      headers: { "X-Primora-Key": API_KEY },
+      responseCallback: http.expectedStatuses(200, 429),
+    },
+  );
+  check(res, { "events 200 or throttled": (r) => r.status === 200 || r.status === 429 });
+}
@@ -0,0 +1,47 @@
+#!/usr/bin/env bash
+# Primora restore: rebuilds the database and storage volume from a backup
+# tarball produced by scripts/backup.sh.
+#
+#   ./scripts/restore.sh backups/primora-20260101-120000.tar.gz
+#
+# DESTRUCTIVE: drops and recreates the target database's objects and
+# overwrites the storage volume. Stop the stack first:
+#   docker compose stop backend auth
+# Then restore, then `docker compose up -d`.
+
+set -euo pipefail
+
+ARCHIVE="${1:?usage: restore.sh <backup.tar.gz>}"
+POSTGRES_CONTAINER="${POSTGRES_CONTAINER:-primora-postgres-1}"
+POSTGRES_USER="${POSTGRES_USER:-primora}"
+POSTGRES_DB="${POSTGRES_DB:-primora}"
+STORAGE_VOLUME="${STORAGE_VOLUME:-primora_backend_storage}"
+
+[ -f "$ARCHIVE" ] || { echo "no such archive: $ARCHIVE" >&2; exit 1; }
+docker exec "$POSTGRES_CONTAINER" true 2>/dev/null \
+  || { echo "postgres container $POSTGRES_CONTAINER is not running" >&2; exit 1; }
+
+echo "This will overwrite database '$POSTGRES_DB' in $POSTGRES_CONTAINER"
+echo "and the contents of volume $STORAGE_VOLUME."
+read -r -p "Continue? [y/N] " confirm
+[ "$confirm" = "y" ] || [ "$confirm" = "Y" ] || { echo "aborted"; exit 1; }
+
+# Workdir under cwd — /tmp is not bind-mountable on every docker setup.
+work="$(mktemp -d "$(pwd)/.restore-tmp.XXXXXX")"
+trap 'rm -rf "$work"' EXIT
+tar -xzf "$ARCHIVE" -C "$work"
+[ -f "$work/db.dump" ] || { echo "archive has no db.dump" >&2; exit 1; }
+
+echo "==> restoring database"
+cat "$work/db.dump" | docker exec -i "$POSTGRES_CONTAINER" \
+  pg_restore -U "$POSTGRES_USER" -d "$POSTGRES_DB" --clean --if-exists --no-owner
+
+if [ -f "$work/storage.tar" ]; then
+  echo "==> restoring storage volume $STORAGE_VOLUME"
+  docker run --rm -v "$STORAGE_VOLUME:/data" -v "$work:/backup" \
+    alpine:3 sh -c 'rm -rf /data/* /data/..?* /data/.[!.]* 2>/dev/null; tar -C /data -xf /backup/storage.tar'
+else
+  echo "warning: archive has no storage.tar — skipping volume" >&2
+fi
+
+echo "==> restore complete; restart the stack with: docker compose up -d"
@@ -26,15 +26,18 @@ if [ ! -f .env ]; then
    echo "🔐 Generating secure secrets..."
    JWT_SECRET=$(openssl rand -base64 32)
    BETTER_AUTH_SECRET=$(openssl rand -base64 32)
+    ENCRYPTION_KEY=$(openssl rand -hex 32)
    
    # Update .env with secrets
    # Use different sed approach for better compatibility
    if [[ "$OSTYPE" == "darwin"* ]]; then
        sed -i '' "s/JWT_SECRET=change-me-super-long-jwt-secret/JWT_SECRET=$JWT_SECRET/g" .env
        sed -i '' "s/BETTER_AUTH_SECRET=change-me-super-long-better-auth-secret/BETTER_AUTH_SECRET=$BETTER_AUTH_SECRET/g" .env
+        sed -i '' "s/^PRIMORA_ENCRYPTION_KEY=.*/PRIMORA_ENCRYPTION_KEY=$ENCRYPTION_KEY/" .env
    else
        sed -i "s/JWT_SECRET=change-me-super-long-jwt-secret/JWT_SECRET=$JWT_SECRET/g" .env
        sed -i "s/BETTER_AUTH_SECRET=change-me-super-long-better-auth-secret/BETTER_AUTH_SECRET=$BETTER_AUTH_SECRET/g" .env
+        sed -i "s/^PRIMORA_ENCRYPTION_KEY=.*/PRIMORA_ENCRYPTION_KEY=$ENCRYPTION_KEY/" .env
    fi
    echo "✅ .env file created and secured."
 else
@@ -55,12 +58,13 @@ else
    sed -i "s/VITE_APP_URL=http:\/\/localhost/VITE_APP_URL=http:\/\/$DOMAIN/g" .env
 fi

-# Start services
+# Start services — the dev overlay adds mailpit for local email capture.
+# Production deploys use the base file alone: docker compose up -d
 echo "📦 Pulling images and starting services..."
 if command -v docker-compose &> /dev/null; then
-    docker-compose up -d
+    docker-compose -f docker-compose.yml -f docker-compose.dev.yml up -d
 else
-    docker compose up -d
+    docker compose -f docker-compose.yml -f docker-compose.dev.yml up -d
 fi

 echo ""
@@ -0,0 +1,102 @@
+#!/usr/bin/env bash
+# End-to-end smoke test against a running Primora stack.
+# Expects the compose stack on BASE_URL (nginx :80 by default).
+#
+#   ./scripts/smoke-e2e.sh
+#   BASE_URL=http://localhost:8080 ./scripts/smoke-e2e.sh
+#
+# Covers: health, signup/login, bootstrap, scoped API keys, ingest,
+# telemetry read, and the Prometheus endpoint. Exits non-zero on failure.
+
+set -euo pipefail
+
+BASE="${BASE_URL:-http://localhost}"
+EMAIL="smoke-$(date +%s)@example.com"
+PASSWORD="smoke-password-$(date +%s)"
+JAR="$(mktemp)"
+trap 'rm -f "$JAR"' EXIT
+
+fail() { echo "FAIL: $*" >&2; exit 1; }
+step() { echo "==> $*"; }
+
+wait_for() {
+  local url="$1" name="$2" tries=60
+  while (( tries-- > 0 )); do
+    if curl -fsS -o /dev/null "$url" 2>/dev/null; then
+      echo "    $name ready"
+      return 0
+    fi
+    sleep 2
+  done
+  fail "$name did not become ready ($url)"
+}
+
+step "waiting for services"
+wait_for "$BASE/api/v1/health/liveness" "backend liveness"
+wait_for "$BASE/api/v1/health/readiness" "backend readiness"
+wait_for "$BASE/auth-health" "auth"
+
+step "signup $EMAIL"
+code=$(curl -sS -o /dev/null -w '%{http_code}' -c "$JAR" \
+  -H 'content-type: application/json' \
+  -d "{\"email\":\"$EMAIL\",\"password\":\"$PASSWORD\",\"name\":\"Smoke Test\"}" \
+  "$BASE/auth/sign-up/email")
+[ "$code" = "200" ] || [ "$code" = "201" ] || fail "sign-up returned $code"
+
+step "fetch api token"
+token=$(curl -fsS -b "$JAR" "$BASE/auth/token" | sed -n 's/.*"token":"\([^"]*\)".*/\1/p')
+[ -n "$token" ] || fail "no token in /auth/token response"
+
+step "bootstrap organization + project"
+code=$(curl -sS -o /tmp/smoke-bootstrap.json -w '%{http_code}' -H "authorization: Bearer $token" -H 'content-type: application/json' \
+  -d '{"organizationName":"Smoke Org","organizationSlug":"smoke-org","projectName":"Smoke Project","projectSlug":"smoke-project"}' \
+  "$BASE/api/v1/bootstrap")
+if [ "$code" = "409" ]; then
+  # Instance already bootstrapped — create a fresh org + project for this user.
+  org_id=$(curl -fsS -H "authorization: Bearer $token" -H 'content-type: application/json' \
+    -d "{\"name\":\"Smoke Org $(date +%s)\",\"slug\":\"smoke-$(date +%s)\"}" \
+    "$BASE/api/v1/organizations" | sed -n 's/.*"id":"\([^"]*\)".*/\1/p')
+  [ -n "$org_id" ] || fail "could not create organization"
+  project_id=$(curl -fsS -H "authorization: Bearer $token" -H 'content-type: application/json' \
+    -d "{\"name\":\"Smoke Project\",\"slug\":\"smoke-$(date +%s)-p\"}" \
+    "$BASE/api/v1/organizations/$org_id/projects" | sed -n 's/.*"id":"\([^"]*\)".*/\1/p')
+else
+  [ "$code" = "200" ] || [ "$code" = "201" ] || { cat /tmp/smoke-bootstrap.json >&2; fail "bootstrap returned $code"; }
+  project_id=$(sed -n 's/.*"project_id":"\([^"]*\)".*/\1/p' /tmp/smoke-bootstrap.json)
+fi
+[ -n "$project_id" ] || fail "no project_id available"
+echo "    project $project_id"
+
+step "create ingest+read API key"
+key_json=$(curl -fsS -H "authorization: Bearer $token" -H 'content-type: application/json' \
+  -d '{"name":"smoke-ingest","scopes":["ingest","read"]}' \
+  "$BASE/api/v1/projects/$project_id/api-keys")
+api_key=$(echo "$key_json" | sed -n 's/.*"secret":"\([^"]*\)".*/\1/p')
+[ -n "$api_key" ] || fail "no secret in api key response"
+echo "$key_json" | grep -q '"ingest"' || fail "scopes missing from api key response"
+
+step "ingest event via X-Primora-Key"
+curl -fsS -o /dev/null -H "X-Primora-Key: $api_key" -H 'content-type: application/json' \
+  -d '{"events":[{"type":"metric","severity":"info","component":"smoke","message":"smoke metric","payload":{"name":"smoke.count","value":1}}]}' \
+  "$BASE/api/v1/ingest"
+
+step "negative: read-only key cannot ingest"
+ro_json=$(curl -fsS -H "authorization: Bearer $token" -H 'content-type: application/json' \
+  -d '{"name":"smoke-readonly","scopes":["read"]}' \
+  "$BASE/api/v1/projects/$project_id/api-keys")
+ro_key=$(echo "$ro_json" | sed -n 's/.*"secret":"\([^"]*\)".*/\1/p')
+code=$(curl -sS -o /dev/null -w '%{http_code}' -H "X-Primora-Key: $ro_key" -H 'content-type: application/json' \
+  -d '{"type":"metric","severity":"info","message":"should be rejected"}' \
+  "$BASE/api/v1/ingest")
+[ "$code" = "403" ] || fail "read-only key ingest returned $code, expected 403"
+
+step "read back events"
+curl -fsS -o /dev/null -H "authorization: Bearer $token" \
+  "$BASE/api/v1/projects/$project_id/events?limit=5"
+
+step "prometheus metrics"
+curl -fsS "$BASE/api/v1/metrics" | grep -q 'primora_http_requests_total' \
+  || fail "metrics endpoint missing primora_http_requests_total"
+
+echo
+echo "smoke: all checks passed"
@@ -20,20 +20,20 @@ FAILED=0
 check_file() {
    if [ -f "$1" ]; then
        echo -e "${GREEN}✓${NC} $2"
-        ((PASSED++))
+        ((++PASSED))
    else
        echo -e "${RED}✗${NC} $2 (missing: $1)"
-        ((FAILED++))
+        ((++FAILED))
    fi
 }

 check_dir() {
    if [ -d "$1" ]; then
        echo -e "${GREEN}✓${NC} $2"
-        ((PASSED++))
+        ((++PASSED))
    else
        echo -e "${RED}✗${NC} $2 (missing: $1)"
-        ((FAILED++))
+        ((++FAILED))
    fi
 }

@@ -58,8 +58,18 @@ echo ""
 echo "📚 Documentation:"
 check_file "PRODUCTION_READINESS.md" "Production readiness checklist"
 check_file "DEPLOYMENT_GUIDE.md" "Deployment guide"
-check_file "PRODUCTION_IMPROVEMENTS.md" "Improvements summary"
-check_file "PRODUCTION_READY_SUMMARY.md" "Production ready summary"
+check_file "ROADMAP.md" "Roadmap"
+echo ""
+
+echo "🛡️ Phase 6 hardening:"
+check_file "apps/backend/db/migrations/00006_api_key_scopes.sql" "API key scopes migration"
+check_file "apps/backend/db/migrations/00007_retention.sql" "Retention migration"
+check_file "apps/backend/internal/services/retention.go" "Retention sweeper"
+check_file "docker-compose.dev.yml" "Dev overlay (Mailpit isolation)"
+check_file "scripts/smoke-e2e.sh" "E2E smoke script"
+check_file "scripts/backup.sh" "Backup script"
+check_file "scripts/restore.sh" "Restore script"
+check_file "scripts/load-sanity.js" "k6 load sanity script"
 echo ""

 echo "🐳 Infrastructure:"