feat: phase 6 production hardening + DB-to-DB transfer

Security:
- Graceful shutdown: http.Server + signal.NotifyContext, 10s drain,
  retention sweeper cancelled before pool close
- Scoped API keys (migration 00006): ingest/read/write/admin enforced in
  auth middleware; scope picker + badges in Settings UI; scopeless legacy
  keys act as admin. Read-only keys now get 403 on ingest (verified live)
- nginx: baseline security headers on all responses, CSP scoped to SPA;
  fixed add_header inheritance shadowing
- better-auth rate limit enabled unconditionally (was gated on NODE_ENV,
  which the shipped .env sets to development)
- Mailpit moved to docker-compose.dev.yml; prod compose cannot expose it
- setup.sh generates PRIMORA_ENCRYPTION_KEY

Reliability/ops:
- Retention: per-project retention_{events,audit,webhook}_days
  (migration 00007, 0=off) + hourly RetentionSweeper; edit UI in Projects
- GET /api/v1/metrics Prometheus exposition
- compose: healthchecks + restart policies everywhere, dragonfly pinned
  to v2.0.0, depends_on service_healthy
- backup.sh/restore.sh (pg_dump -Fc + storage volume); restore verified
  against scratch postgres+volume
- .dockerignore: exclude target dirs (build context was 4.7GB)

Correctness/testing:
- DBX transfer endpoint: POST /db-connections/:id/transfer — SQL or Redis
  target, 1000-row bound, server-side creds (verified live pg→dragonfly)
- Pagination audit: ListComponents capped at 500 (auto-registers)
- CI: goose up/reset/up job + compose e2e-smoke job (smoke-e2e.sh
  verified locally: signup, bootstrap, scoped keys, ingest, metrics)
- k6 load sanity script: 6650 reqs, ~217/s, 0 real failures, p95 221ms
  (429s = configured rate limiter working)

Docs: DEPLOYMENT_GUIDE.md, PRODUCTION_READINESS.md; verify script
repaired (stale refs + set -e arithmetic abort). ROADMAP: phase 2 +
phase 6 marked complete.

Authored By: TDvorak <[email protected]>

Generated with [Devin](https://devin.ai)

Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com>

apps/backend/internal/database/db/api_keys.sql.go

@@ -18,9 +18,10 @@ INSERT INTO core.api_keys (
   name,
   prefix,
   secret_hash,
+  scopes,
   created_by_user_id
-) VALUES ($1, $2, $3, $4, $5)
-RETURNING id, project_id, name, prefix, secret_hash, created_by_user_id, last_used_at, revoked_at, created_at
+) VALUES ($1, $2, $3, $4, $5, $6)
+RETURNING id, project_id, name, prefix, secret_hash, created_by_user_id, last_used_at, revoked_at, created_at, scopes
 `
 
 type CreateAPIKeyParams struct {
@@ -28,6 +29,7 @@ type CreateAPIKeyParams struct {
 	Name            string      `json:"name"`
 	Prefix          string      `json:"prefix"`
 	SecretHash      []byte      `json:"secret_hash"`
+	Scopes          []string    `json:"scopes"`
 	CreatedByUserID pgtype.UUID `json:"created_by_user_id"`
 }
 
@@ -37,6 +39,7 @@ func (q *Queries) CreateAPIKey(ctx context.Context, arg CreateAPIKeyParams) (Cor
 		arg.Name,
 		arg.Prefix,
 		arg.SecretHash,
+		arg.Scopes,
 		arg.CreatedByUserID,
 	)
 	var i CoreApiKey
@@ -50,12 +53,13 @@ func (q *Queries) CreateAPIKey(ctx context.Context, arg CreateAPIKeyParams) (Cor
 		&i.LastUsedAt,
 		&i.RevokedAt,
 		&i.CreatedAt,
+		&i.Scopes,
 	)
 	return i, err
 }
 
 const getAPIKeyByIDForProject = `-- name: GetAPIKeyByIDForProject :one
-SELECT id, project_id, name, prefix, secret_hash, created_by_user_id, last_used_at, revoked_at, created_at FROM core.api_keys
+SELECT id, project_id, name, prefix, secret_hash, created_by_user_id, last_used_at, revoked_at, created_at, scopes FROM core.api_keys
 WHERE project_id = $1
   AND id = $2
 `
@@ -78,13 +82,14 @@ func (q *Queries) GetAPIKeyByIDForProject(ctx context.Context, arg GetAPIKeyByID
 		&i.LastUsedAt,
 		&i.RevokedAt,
 		&i.CreatedAt,
+		&i.Scopes,
 	)
 	return i, err
 }
 
 const getAPIKeyByPrefix = `-- name: GetAPIKeyByPrefix :one
 SELECT
-  ak.id, ak.project_id, ak.name, ak.prefix, ak.secret_hash, ak.created_by_user_id, ak.last_used_at, ak.revoked_at, ak.created_at,
+  ak.id, ak.project_id, ak.name, ak.prefix, ak.secret_hash, ak.created_by_user_id, ak.last_used_at, ak.revoked_at, ak.created_at, ak.scopes,
   p.organization_id
 FROM core.api_keys ak
 JOIN core.projects p ON p.id = ak.project_id
@@ -101,6 +106,7 @@ type GetAPIKeyByPrefixRow struct {
 	LastUsedAt      pgtype.Timestamptz `json:"last_used_at"`
 	RevokedAt       pgtype.Timestamptz `json:"revoked_at"`
 	CreatedAt       pgtype.Timestamptz `json:"created_at"`
+	Scopes          []string           `json:"scopes"`
 	OrganizationID  uuid.UUID          `json:"organization_id"`
 }
 
@@ -117,13 +123,14 @@ func (q *Queries) GetAPIKeyByPrefix(ctx context.Context, prefix string) (GetAPIK
 		&i.LastUsedAt,
 		&i.RevokedAt,
 		&i.CreatedAt,
+		&i.Scopes,
 		&i.OrganizationID,
 	)
 	return i, err
 }
 
 const listAPIKeysForProject = `-- name: ListAPIKeysForProject :many
-SELECT id, project_id, name, prefix, secret_hash, created_by_user_id, last_used_at, revoked_at, created_at FROM core.api_keys
+SELECT id, project_id, name, prefix, secret_hash, created_by_user_id, last_used_at, revoked_at, created_at, scopes FROM core.api_keys
 WHERE project_id = $1
 ORDER BY created_at DESC
 `
@@ -147,6 +154,7 @@ func (q *Queries) ListAPIKeysForProject(ctx context.Context, projectID uuid.UUID
 			&i.LastUsedAt,
 			&i.RevokedAt,
 			&i.CreatedAt,
+			&i.Scopes,
 		); err != nil {
 			return nil, err
 		}
@@ -164,7 +172,7 @@ SET revoked_at = NOW()
 WHERE project_id = $1
   AND id = $2
   AND revoked_at IS NULL
-RETURNING id, project_id, name, prefix, secret_hash, created_by_user_id, last_used_at, revoked_at, created_at
+RETURNING id, project_id, name, prefix, secret_hash, created_by_user_id, last_used_at, revoked_at, created_at, scopes
 `
 
 type RevokeAPIKeyParams struct {
@@ -185,6 +193,7 @@ func (q *Queries) RevokeAPIKey(ctx context.Context, arg RevokeAPIKeyParams) (Cor
 		&i.LastUsedAt,
 		&i.RevokedAt,
 		&i.CreatedAt,
+		&i.Scopes,
 	)
 	return i, err
 }

apps/backend/internal/database/db/bootstrap.sql.go

@@ -24,7 +24,7 @@ new_org_member AS (
 new_project AS (
   INSERT INTO core.projects (organization_id, slug, name, description)
   SELECT id, $4, $5, $6 FROM new_org
-  RETURNING id, organization_id, slug, name, description, created_at
+  RETURNING id, organization_id, slug, name, description, created_at, retention_events_days, retention_audit_days, retention_webhook_days
 ),
 new_project_member AS (
   INSERT INTO core.project_members (project_id, user_id, role)

apps/backend/internal/database/db/models.go

@@ -150,6 +150,7 @@ type CoreApiKey struct {
 	LastUsedAt      pgtype.Timestamptz `json:"last_used_at"`
 	RevokedAt       pgtype.Timestamptz `json:"revoked_at"`
 	CreatedAt       pgtype.Timestamptz `json:"created_at"`
+	Scopes          []string           `json:"scopes"`
 }
 
 type CoreAuditLog struct {
@@ -273,12 +274,15 @@ type CoreOrganizationMember struct {
 }
 
 type CoreProject struct {
-	ID             uuid.UUID          `json:"id"`
-	OrganizationID uuid.UUID          `json:"organization_id"`
-	Slug           string             `json:"slug"`
-	Name           string             `json:"name"`
-	Description    *string            `json:"description"`
-	CreatedAt      pgtype.Timestamptz `json:"created_at"`
+	ID                   uuid.UUID          `json:"id"`
+	OrganizationID       uuid.UUID          `json:"organization_id"`
+	Slug                 string             `json:"slug"`
+	Name                 string             `json:"name"`
+	Description          *string            `json:"description"`
+	CreatedAt            pgtype.Timestamptz `json:"created_at"`
+	RetentionEventsDays  int32              `json:"retention_events_days"`
+	RetentionAuditDays   int32              `json:"retention_audit_days"`
+	RetentionWebhookDays int32              `json:"retention_webhook_days"`
 }
 
 type CoreProjectInvitation struct {

apps/backend/internal/database/db/projects.sql.go

@@ -59,7 +59,7 @@ INSERT INTO core.projects (
   name,
   description
 ) VALUES ($1, $2, $3, $4)
-RETURNING id, organization_id, slug, name, description, created_at
+RETURNING id, organization_id, slug, name, description, created_at, retention_events_days, retention_audit_days, retention_webhook_days
 `
 
 type CreateProjectParams struct {
@@ -84,6 +84,9 @@ func (q *Queries) CreateProject(ctx context.Context, arg CreateProjectParams) (C
 		&i.Name,
 		&i.Description,
 		&i.CreatedAt,
+		&i.RetentionEventsDays,
+		&i.RetentionAuditDays,
+		&i.RetentionWebhookDays,
 	)
 	return i, err
 }
@@ -91,7 +94,7 @@ func (q *Queries) CreateProject(ctx context.Context, arg CreateProjectParams) (C
 const deleteProjectByID = `-- name: DeleteProjectByID :one
 DELETE FROM core.projects
 WHERE id = $1
-RETURNING id, organization_id, slug, name, description, created_at
+RETURNING id, organization_id, slug, name, description, created_at, retention_events_days, retention_audit_days, retention_webhook_days
 `
 
 func (q *Queries) DeleteProjectByID(ctx context.Context, id uuid.UUID) (CoreProject, error) {
@@ -104,12 +107,15 @@ func (q *Queries) DeleteProjectByID(ctx context.Context, id uuid.UUID) (CoreProj
 		&i.Name,
 		&i.Description,
 		&i.CreatedAt,
+		&i.RetentionEventsDays,
+		&i.RetentionAuditDays,
+		&i.RetentionWebhookDays,
 	)
 	return i, err
 }
 
 const getProjectByID = `-- name: GetProjectByID :one
-SELECT id, organization_id, slug, name, description, created_at FROM core.projects
+SELECT id, organization_id, slug, name, description, created_at, retention_events_days, retention_audit_days, retention_webhook_days FROM core.projects
 WHERE id = $1
 `
 
@@ -123,6 +129,9 @@ func (q *Queries) GetProjectByID(ctx context.Context, id uuid.UUID) (CoreProject
 		&i.Name,
 		&i.Description,
 		&i.CreatedAt,
+		&i.RetentionEventsDays,
+		&i.RetentionAuditDays,
+		&i.RetentionWebhookDays,
 	)
 	return i, err
 }
@@ -327,7 +336,7 @@ func (q *Queries) ListProjectMembers(ctx context.Context, projectID uuid.UUID) (
 
 const listProjectsForOrganization = `-- name: ListProjectsForOrganization :many
 SELECT
-  p.id, p.organization_id, p.slug, p.name, p.description, p.created_at,
+  p.id, p.organization_id, p.slug, p.name, p.description, p.created_at, p.retention_events_days, p.retention_audit_days, p.retention_webhook_days,
   pm.role AS membership_role
 FROM core.projects p
 LEFT JOIN core.project_members pm
@@ -350,13 +359,16 @@ type ListProjectsForOrganizationParams struct {
 }
 
 type ListProjectsForOrganizationRow struct {
-	ID             uuid.UUID           `json:"id"`
-	OrganizationID uuid.UUID           `json:"organization_id"`
-	Slug           string              `json:"slug"`
-	Name           string              `json:"name"`
-	Description    *string             `json:"description"`
-	CreatedAt      pgtype.Timestamptz  `json:"created_at"`
-	MembershipRole NullCoreProjectRole `json:"membership_role"`
+	ID                   uuid.UUID           `json:"id"`
+	OrganizationID       uuid.UUID           `json:"organization_id"`
+	Slug                 string              `json:"slug"`
+	Name                 string              `json:"name"`
+	Description          *string             `json:"description"`
+	CreatedAt            pgtype.Timestamptz  `json:"created_at"`
+	RetentionEventsDays  int32               `json:"retention_events_days"`
+	RetentionAuditDays   int32               `json:"retention_audit_days"`
+	RetentionWebhookDays int32               `json:"retention_webhook_days"`
+	MembershipRole       NullCoreProjectRole `json:"membership_role"`
 }
 
 func (q *Queries) ListProjectsForOrganization(ctx context.Context, arg ListProjectsForOrganizationParams) ([]ListProjectsForOrganizationRow, error) {
@@ -375,6 +387,9 @@ func (q *Queries) ListProjectsForOrganization(ctx context.Context, arg ListProje
 			&i.Name,
 			&i.Description,
 			&i.CreatedAt,
+			&i.RetentionEventsDays,
+			&i.RetentionAuditDays,
+			&i.RetentionWebhookDays,
 			&i.MembershipRole,
 		); err != nil {
 			return nil, err
@@ -416,16 +431,22 @@ const updateProjectByID = `-- name: UpdateProjectByID :one
 UPDATE core.projects
 SET slug = $2,
     name = $3,
-    description = $4
+    description = $4,
+    retention_events_days = COALESCE($5, retention_events_days),
+    retention_audit_days = COALESCE($6, retention_audit_days),
+    retention_webhook_days = COALESCE($7, retention_webhook_days)
 WHERE id = $1
-RETURNING id, organization_id, slug, name, description, created_at
+RETURNING id, organization_id, slug, name, description, created_at, retention_events_days, retention_audit_days, retention_webhook_days
 `
 
 type UpdateProjectByIDParams struct {
-	ID          uuid.UUID `json:"id"`
-	Slug        string    `json:"slug"`
-	Name        string    `json:"name"`
-	Description *string   `json:"description"`
+	ID                   uuid.UUID `json:"id"`
+	Slug                 string    `json:"slug"`
+	Name                 string    `json:"name"`
+	Description          *string   `json:"description"`
+	RetentionEventsDays  *int32    `json:"retention_events_days"`
+	RetentionAuditDays   *int32    `json:"retention_audit_days"`
+	RetentionWebhookDays *int32    `json:"retention_webhook_days"`
 }
 
 func (q *Queries) UpdateProjectByID(ctx context.Context, arg UpdateProjectByIDParams) (CoreProject, error) {
@@ -434,6 +455,9 @@ func (q *Queries) UpdateProjectByID(ctx context.Context, arg UpdateProjectByIDPa
 		arg.Slug,
 		arg.Name,
 		arg.Description,
+		arg.RetentionEventsDays,
+		arg.RetentionAuditDays,
+		arg.RetentionWebhookDays,
 	)
 	var i CoreProject
 	err := row.Scan(
@@ -443,6 +467,9 @@ func (q *Queries) UpdateProjectByID(ctx context.Context, arg UpdateProjectByIDPa
 		&i.Name,
 		&i.Description,
 		&i.CreatedAt,
+		&i.RetentionEventsDays,
+		&i.RetentionAuditDays,
+		&i.RetentionWebhookDays,
 	)
 	return i, err
 }

apps/backend/internal/database/db/querier.go

@@ -100,6 +100,15 @@ type Querier interface {
 	RemoveProjectMember(ctx context.Context, arg RemoveProjectMemberParams) (CoreProjectMember, error)
 	RemoveProjectMembershipsForOrganizationUser(ctx context.Context, arg RemoveProjectMembershipsForOrganizationUserParams) error
 	RevokeAPIKey(ctx context.Context, arg RevokeAPIKeyParams) (CoreApiKey, error)
+	SweepExpiredAuditLogs(ctx context.Context) (int64, error)
+	// Per-project retention sweeps. A retention of 0 days disables the sweep for
+	// that stream. Pending webhook deliveries are never swept — they may still be
+	// retrying.
+	SweepExpiredEvents(ctx context.Context) (int64, error)
+	// Org-scoped audit rows (project_id NULL) have no project retention — sweep
+	// them on a fixed 365-day floor so they cannot grow unbounded.
+	SweepExpiredOrgAuditLogs(ctx context.Context) (int64, error)
+	SweepExpiredWebhookDeliveries(ctx context.Context) (int64, error)
 	TouchAPIKey(ctx context.Context, id uuid.UUID) error
 	UpdateBucketByID(ctx context.Context, arg UpdateBucketByIDParams) (CoreBucket, error)
 	UpdateCollection(ctx context.Context, arg UpdateCollectionParams) (CoreCollection, error)

apps/backend/internal/database/db/retention.sql.go

@@ -0,0 +1,80 @@
+// Code generated by sqlc. DO NOT EDIT.
+// versions:
+//   sqlc v1.29.0
+// source: retention.sql
+
+package db
+
+import (
+	"context"
+)
+
+const sweepExpiredAuditLogs = `-- name: SweepExpiredAuditLogs :execrows
+DELETE FROM core.audit_logs a
+USING core.projects p
+WHERE a.project_id = p.id
+  AND p.retention_audit_days > 0
+  AND a.created_at < NOW() - make_interval(days => p.retention_audit_days)
+`
+
+func (q *Queries) SweepExpiredAuditLogs(ctx context.Context) (int64, error) {
+	result, err := q.db.Exec(ctx, sweepExpiredAuditLogs)
+	if err != nil {
+		return 0, err
+	}
+	return result.RowsAffected(), nil
+}
+
+const sweepExpiredEvents = `-- name: SweepExpiredEvents :execrows
+
+DELETE FROM core.events e
+USING core.projects p
+WHERE e.project_id = p.id
+  AND p.retention_events_days > 0
+  AND e.ts < NOW() - make_interval(days => p.retention_events_days)
+`
+
+// Per-project retention sweeps. A retention of 0 days disables the sweep for
+// that stream. Pending webhook deliveries are never swept — they may still be
+// retrying.
+func (q *Queries) SweepExpiredEvents(ctx context.Context) (int64, error) {
+	result, err := q.db.Exec(ctx, sweepExpiredEvents)
+	if err != nil {
+		return 0, err
+	}
+	return result.RowsAffected(), nil
+}
+
+const sweepExpiredOrgAuditLogs = `-- name: SweepExpiredOrgAuditLogs :execrows
+DELETE FROM core.audit_logs a
+WHERE a.project_id IS NULL
+  AND a.created_at < NOW() - INTERVAL '365 days'
+`
+
+// Org-scoped audit rows (project_id NULL) have no project retention — sweep
+// them on a fixed 365-day floor so they cannot grow unbounded.
+func (q *Queries) SweepExpiredOrgAuditLogs(ctx context.Context) (int64, error) {
+	result, err := q.db.Exec(ctx, sweepExpiredOrgAuditLogs)
+	if err != nil {
+		return 0, err
+	}
+	return result.RowsAffected(), nil
+}
+
+const sweepExpiredWebhookDeliveries = `-- name: SweepExpiredWebhookDeliveries :execrows
+DELETE FROM core.webhook_deliveries d
+USING core.webhooks w
+JOIN core.projects p ON p.id = w.project_id
+WHERE d.webhook_id = w.id
+  AND p.retention_webhook_days > 0
+  AND d.status <> 'pending'
+  AND d.created_at < NOW() - make_interval(days => p.retention_webhook_days)
+`
+
+func (q *Queries) SweepExpiredWebhookDeliveries(ctx context.Context) (int64, error) {
+	result, err := q.db.Exec(ctx, sweepExpiredWebhookDeliveries)
+	if err != nil {
+		return 0, err
+	}
+	return result.RowsAffected(), nil
+}

apps/backend/internal/database/db/telemetry.sql.go

@@ -239,6 +239,7 @@ const listComponents = `-- name: ListComponents :many
 SELECT id, project_id, name, kind, meta, created_at FROM core.components
 WHERE project_id = $1
 ORDER BY name
+LIMIT 500
 `
 
 func (q *Queries) ListComponents(ctx context.Context, projectID uuid.UUID) ([]CoreComponent, error) {