feat: phase 6 production hardening + DB-to-DB transfer

Security:
- Graceful shutdown: http.Server + signal.NotifyContext, 10s drain,
  retention sweeper cancelled before pool close
- Scoped API keys (migration 00006): ingest/read/write/admin enforced in
  auth middleware; scope picker + badges in Settings UI; scopeless legacy
  keys act as admin. Read-only keys now get 403 on ingest (verified live)
- nginx: baseline security headers on all responses, CSP scoped to SPA;
  fixed add_header inheritance shadowing
- better-auth rate limit enabled unconditionally (was gated on NODE_ENV,
  which the shipped .env sets to development)
- Mailpit moved to docker-compose.dev.yml; prod compose cannot expose it
- setup.sh generates PRIMORA_ENCRYPTION_KEY

Reliability/ops:
- Retention: per-project retention_{events,audit,webhook}_days
  (migration 00007, 0=off) + hourly RetentionSweeper; edit UI in Projects
- GET /api/v1/metrics Prometheus exposition
- compose: healthchecks + restart policies everywhere, dragonfly pinned
  to v2.0.0, depends_on service_healthy
- backup.sh/restore.sh (pg_dump -Fc + storage volume); restore verified
  against scratch postgres+volume
- .dockerignore: exclude target dirs (build context was 4.7GB)

Correctness/testing:
- DBX transfer endpoint: POST /db-connections/:id/transfer — SQL or Redis
  target, 1000-row bound, server-side creds (verified live pg→dragonfly)
- Pagination audit: ListComponents capped at 500 (auto-registers)
- CI: goose up/reset/up job + compose e2e-smoke job (smoke-e2e.sh
  verified locally: signup, bootstrap, scoped keys, ingest, metrics)
- k6 load sanity script: 6650 reqs, ~217/s, 0 real failures, p95 221ms
  (429s = configured rate limiter working)

Docs: DEPLOYMENT_GUIDE.md, PRODUCTION_READINESS.md; verify script
repaired (stale refs + set -e arithmetic abort). ROADMAP: phase 2 +
phase 6 marked complete.

Authored By: TDvorak <[email protected]>

Generated with [Devin](https://devin.ai)

Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com>

apps/auth/src/lib/auth.ts

@@ -75,6 +75,12 @@ export const auth = betterAuth({
       });
     },
   },
+  // Enabled unconditionally — better-auth only turns this on when NODE_ENV is
+  // production, but the shipped .env defaults to development. Credential
+  // endpoints get 3 req/10s; the IP-level limiter in index.ts sits in front.
+  rateLimit: {
+    enabled: true,
+  },
   socialProviders,
   plugins: [
     jwt({

apps/backend/db/migrations/00006_api_key_scopes.sql

@@ -0,0 +1,6 @@
+-- +goose Up
+ALTER TABLE core.api_keys
+  ADD COLUMN scopes TEXT[] NOT NULL DEFAULT '{admin}';
+
+-- +goose Down
+ALTER TABLE core.api_keys DROP COLUMN IF EXISTS scopes;

apps/backend/db/migrations/00007_retention.sql

@@ -0,0 +1,11 @@
+-- +goose Up
+ALTER TABLE core.projects
+  ADD COLUMN retention_events_days INT NOT NULL DEFAULT 90,
+  ADD COLUMN retention_audit_days INT NOT NULL DEFAULT 365,
+  ADD COLUMN retention_webhook_days INT NOT NULL DEFAULT 30;
+
+-- +goose Down
+ALTER TABLE core.projects
+  DROP COLUMN IF EXISTS retention_events_days,
+  DROP COLUMN IF EXISTS retention_audit_days,
+  DROP COLUMN IF EXISTS retention_webhook_days;

apps/backend/db/queries/api_keys.sql

@@ -4,8 +4,9 @@ INSERT INTO core.api_keys (
   name,
   prefix,
   secret_hash,
+  scopes,
   created_by_user_id
-) VALUES ($1, $2, $3, $4, $5)
+) VALUES ($1, $2, $3, $4, $5, $6)
 RETURNING *;
 
 -- name: ListAPIKeysForProject :many

apps/backend/db/queries/projects.sql

@@ -28,7 +28,10 @@ RETURNING *;
 UPDATE core.projects
 SET slug = $2,
     name = $3,
-    description = $4
+    description = $4,
+    retention_events_days = COALESCE(sqlc.narg('retention_events_days'), retention_events_days),
+    retention_audit_days = COALESCE(sqlc.narg('retention_audit_days'), retention_audit_days),
+    retention_webhook_days = COALESCE(sqlc.narg('retention_webhook_days'), retention_webhook_days)
 WHERE id = $1
 RETURNING *;
 

apps/backend/db/queries/retention.sql

@@ -0,0 +1,33 @@
+-- Per-project retention sweeps. A retention of 0 days disables the sweep for
+-- that stream. Pending webhook deliveries are never swept — they may still be
+-- retrying.
+
+-- name: SweepExpiredEvents :execrows
+DELETE FROM core.events e
+USING core.projects p
+WHERE e.project_id = p.id
+  AND p.retention_events_days > 0
+  AND e.ts < NOW() - make_interval(days => p.retention_events_days);
+
+-- name: SweepExpiredAuditLogs :execrows
+DELETE FROM core.audit_logs a
+USING core.projects p
+WHERE a.project_id = p.id
+  AND p.retention_audit_days > 0
+  AND a.created_at < NOW() - make_interval(days => p.retention_audit_days);
+
+-- name: SweepExpiredOrgAuditLogs :execrows
+-- Org-scoped audit rows (project_id NULL) have no project retention — sweep
+-- them on a fixed 365-day floor so they cannot grow unbounded.
+DELETE FROM core.audit_logs a
+WHERE a.project_id IS NULL
+  AND a.created_at < NOW() - INTERVAL '365 days';
+
+-- name: SweepExpiredWebhookDeliveries :execrows
+DELETE FROM core.webhook_deliveries d
+USING core.webhooks w
+JOIN core.projects p ON p.id = w.project_id
+WHERE d.webhook_id = w.id
+  AND p.retention_webhook_days > 0
+  AND d.status <> 'pending'
+  AND d.created_at < NOW() - make_interval(days => p.retention_webhook_days);

apps/backend/db/queries/telemetry.sql

@@ -11,7 +11,8 @@ WHERE project_id = $1 AND name = $2;
 -- name: ListComponents :many
 SELECT * FROM core.components
 WHERE project_id = $1
-ORDER BY name;
+ORDER BY name
+LIMIT 500;
 
 -- name: DeleteComponent :exec
 DELETE FROM core.components

apps/backend/internal/app/app.go

@@ -2,12 +2,16 @@ package app
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"log/slog"
+	"net/http"
 	"net/url"
 	"os"
+	"os/signal"
 	"strconv"
 	"strings"
+	"syscall"
 	"time"
 
 	"github.com/gin-gonic/gin"
@@ -37,6 +41,8 @@ type App struct {
 	Redis    *redis.Client
 	DBX      *dbx.Client
 	Platform *services.PlatformService
+
+	sweepCancel context.CancelFunc
 }
 
 func Bootstrap(ctx context.Context) (*App, error) {
@@ -157,21 +163,53 @@ func Bootstrap(ctx context.Context) (*App, error) {
 	}
 	handler.Register(router)
 
+	sweepCtx, sweepCancel := context.WithCancel(context.Background())
+	sweeper := services.NewRetentionSweeper(repo, logger)
+	go sweeper.Run(sweepCtx)
+
 	return &App{
-		Config:   cfg,
-		Router:   router,
-		Logger:   logger,
-		DB:       dbPool,
-		Redis:    redisClient,
-		DBX:      dbxClient,
-		Platform: platform,
+		Config:      cfg,
+		Router:      router,
+		Logger:      logger,
+		DB:          dbPool,
+		Redis:       redisClient,
+		DBX:         dbxClient,
+		Platform:    platform,
+		sweepCancel: sweepCancel,
 	}, nil
 }
 
 func (a *App) Run() error {
 	address := ":" + a.Config.ServerPort
-	a.Logger.Info("primora backend starting", "address", address)
-	return a.Router.Run(address)
+	srv := &http.Server{Addr: address, Handler: a.Router}
+
+	ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
+	defer stop()
+
+	errCh := make(chan error, 1)
+	go func() {
+		a.Logger.Info("primora backend starting", "address", address)
+		errCh <- srv.ListenAndServe()
+	}()
+
+	select {
+	case err := <-errCh:
+		return err
+	case <-ctx.Done():
+	}
+
+	a.Logger.Info("shutdown signal received, draining in-flight requests")
+	shutdownCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+	if err := srv.Shutdown(shutdownCtx); err != nil {
+		return fmt.Errorf("graceful shutdown: %w", err)
+	}
+	if err := <-errCh; err != nil && !errors.Is(err, http.ErrServerClosed) {
+		return err
+	}
+	// Stop the retention sweeper before main closes the DB pool.
+	a.sweepCancel()
+	return nil
 }
 
 // firstPartyDBSeeds turns the platform's own Postgres and Dragonfly URLs into
@@ -206,6 +244,9 @@ func firstPartyDBSeeds(cfg config.Config) []services.ManagedDBSeed {
 }
 
 func (a *App) Close() error {
+	if a.sweepCancel != nil {
+		a.sweepCancel()
+	}
 	if a.Platform != nil {
 		a.Platform.Close()
 	}

apps/backend/internal/database/db/api_keys.sql.go

@@ -18,9 +18,10 @@ INSERT INTO core.api_keys (
   name,
   prefix,
   secret_hash,
+  scopes,
   created_by_user_id
-) VALUES ($1, $2, $3, $4, $5)
-RETURNING id, project_id, name, prefix, secret_hash, created_by_user_id, last_used_at, revoked_at, created_at
+) VALUES ($1, $2, $3, $4, $5, $6)
+RETURNING id, project_id, name, prefix, secret_hash, created_by_user_id, last_used_at, revoked_at, created_at, scopes
 `
 
 type CreateAPIKeyParams struct {
@@ -28,6 +29,7 @@ type CreateAPIKeyParams struct {
 	Name            string      `json:"name"`
 	Prefix          string      `json:"prefix"`
 	SecretHash      []byte      `json:"secret_hash"`
+	Scopes          []string    `json:"scopes"`
 	CreatedByUserID pgtype.UUID `json:"created_by_user_id"`
 }
 
@@ -37,6 +39,7 @@ func (q *Queries) CreateAPIKey(ctx context.Context, arg CreateAPIKeyParams) (Cor
 		arg.Name,
 		arg.Prefix,
 		arg.SecretHash,
+		arg.Scopes,
 		arg.CreatedByUserID,
 	)
 	var i CoreApiKey
@@ -50,12 +53,13 @@ func (q *Queries) CreateAPIKey(ctx context.Context, arg CreateAPIKeyParams) (Cor
 		&i.LastUsedAt,
 		&i.RevokedAt,
 		&i.CreatedAt,
+		&i.Scopes,
 	)
 	return i, err
 }
 
 const getAPIKeyByIDForProject = `-- name: GetAPIKeyByIDForProject :one
-SELECT id, project_id, name, prefix, secret_hash, created_by_user_id, last_used_at, revoked_at, created_at FROM core.api_keys
+SELECT id, project_id, name, prefix, secret_hash, created_by_user_id, last_used_at, revoked_at, created_at, scopes FROM core.api_keys
 WHERE project_id = $1
   AND id = $2
 `
@@ -78,13 +82,14 @@ func (q *Queries) GetAPIKeyByIDForProject(ctx context.Context, arg GetAPIKeyByID
 		&i.LastUsedAt,
 		&i.RevokedAt,
 		&i.CreatedAt,
+		&i.Scopes,
 	)
 	return i, err
 }
 
 const getAPIKeyByPrefix = `-- name: GetAPIKeyByPrefix :one
 SELECT
-  ak.id, ak.project_id, ak.name, ak.prefix, ak.secret_hash, ak.created_by_user_id, ak.last_used_at, ak.revoked_at, ak.created_at,
+  ak.id, ak.project_id, ak.name, ak.prefix, ak.secret_hash, ak.created_by_user_id, ak.last_used_at, ak.revoked_at, ak.created_at, ak.scopes,
   p.organization_id
 FROM core.api_keys ak
 JOIN core.projects p ON p.id = ak.project_id
@@ -101,6 +106,7 @@ type GetAPIKeyByPrefixRow struct {
 	LastUsedAt      pgtype.Timestamptz `json:"last_used_at"`
 	RevokedAt       pgtype.Timestamptz `json:"revoked_at"`
 	CreatedAt       pgtype.Timestamptz `json:"created_at"`
+	Scopes          []string           `json:"scopes"`
 	OrganizationID  uuid.UUID          `json:"organization_id"`
 }
 
@@ -117,13 +123,14 @@ func (q *Queries) GetAPIKeyByPrefix(ctx context.Context, prefix string) (GetAPIK
 		&i.LastUsedAt,
 		&i.RevokedAt,
 		&i.CreatedAt,
+		&i.Scopes,
 		&i.OrganizationID,
 	)
 	return i, err
 }
 
 const listAPIKeysForProject = `-- name: ListAPIKeysForProject :many
-SELECT id, project_id, name, prefix, secret_hash, created_by_user_id, last_used_at, revoked_at, created_at FROM core.api_keys
+SELECT id, project_id, name, prefix, secret_hash, created_by_user_id, last_used_at, revoked_at, created_at, scopes FROM core.api_keys
 WHERE project_id = $1
 ORDER BY created_at DESC
 `
@@ -147,6 +154,7 @@ func (q *Queries) ListAPIKeysForProject(ctx context.Context, projectID uuid.UUID
 			&i.LastUsedAt,
 			&i.RevokedAt,
 			&i.CreatedAt,
+			&i.Scopes,
 		); err != nil {
 			return nil, err
 		}
@@ -164,7 +172,7 @@ SET revoked_at = NOW()
 WHERE project_id = $1
   AND id = $2
   AND revoked_at IS NULL
-RETURNING id, project_id, name, prefix, secret_hash, created_by_user_id, last_used_at, revoked_at, created_at
+RETURNING id, project_id, name, prefix, secret_hash, created_by_user_id, last_used_at, revoked_at, created_at, scopes
 `
 
 type RevokeAPIKeyParams struct {
@@ -185,6 +193,7 @@ func (q *Queries) RevokeAPIKey(ctx context.Context, arg RevokeAPIKeyParams) (Cor
 		&i.LastUsedAt,
 		&i.RevokedAt,
 		&i.CreatedAt,
+		&i.Scopes,
 	)
 	return i, err
 }

apps/backend/internal/database/db/bootstrap.sql.go

@@ -24,7 +24,7 @@ new_org_member AS (
 new_project AS (
   INSERT INTO core.projects (organization_id, slug, name, description)
   SELECT id, $4, $5, $6 FROM new_org
-  RETURNING id, organization_id, slug, name, description, created_at
+  RETURNING id, organization_id, slug, name, description, created_at, retention_events_days, retention_audit_days, retention_webhook_days
 ),
 new_project_member AS (
   INSERT INTO core.project_members (project_id, user_id, role)

apps/backend/internal/database/db/models.go

@@ -150,6 +150,7 @@ type CoreApiKey struct {
 	LastUsedAt      pgtype.Timestamptz `json:"last_used_at"`
 	RevokedAt       pgtype.Timestamptz `json:"revoked_at"`
 	CreatedAt       pgtype.Timestamptz `json:"created_at"`
+	Scopes          []string           `json:"scopes"`
 }
 
 type CoreAuditLog struct {
@@ -273,12 +274,15 @@ type CoreOrganizationMember struct {
 }
 
 type CoreProject struct {
-	ID             uuid.UUID          `json:"id"`
-	OrganizationID uuid.UUID          `json:"organization_id"`
-	Slug           string             `json:"slug"`
-	Name           string             `json:"name"`
-	Description    *string            `json:"description"`
-	CreatedAt      pgtype.Timestamptz `json:"created_at"`
+	ID                   uuid.UUID          `json:"id"`
+	OrganizationID       uuid.UUID          `json:"organization_id"`
+	Slug                 string             `json:"slug"`
+	Name                 string             `json:"name"`
+	Description          *string            `json:"description"`
+	CreatedAt            pgtype.Timestamptz `json:"created_at"`
+	RetentionEventsDays  int32              `json:"retention_events_days"`
+	RetentionAuditDays   int32              `json:"retention_audit_days"`
+	RetentionWebhookDays int32              `json:"retention_webhook_days"`
 }
 
 type CoreProjectInvitation struct {

apps/backend/internal/database/db/projects.sql.go

@@ -59,7 +59,7 @@ INSERT INTO core.projects (
   name,
   description
 ) VALUES ($1, $2, $3, $4)
-RETURNING id, organization_id, slug, name, description, created_at
+RETURNING id, organization_id, slug, name, description, created_at, retention_events_days, retention_audit_days, retention_webhook_days
 `
 
 type CreateProjectParams struct {
@@ -84,6 +84,9 @@ func (q *Queries) CreateProject(ctx context.Context, arg CreateProjectParams) (C
 		&i.Name,
 		&i.Description,
 		&i.CreatedAt,
+		&i.RetentionEventsDays,
+		&i.RetentionAuditDays,
+		&i.RetentionWebhookDays,
 	)
 	return i, err
 }
@@ -91,7 +94,7 @@ func (q *Queries) CreateProject(ctx context.Context, arg CreateProjectParams) (C
 const deleteProjectByID = `-- name: DeleteProjectByID :one
 DELETE FROM core.projects
 WHERE id = $1
-RETURNING id, organization_id, slug, name, description, created_at
+RETURNING id, organization_id, slug, name, description, created_at, retention_events_days, retention_audit_days, retention_webhook_days
 `
 
 func (q *Queries) DeleteProjectByID(ctx context.Context, id uuid.UUID) (CoreProject, error) {
@@ -104,12 +107,15 @@ func (q *Queries) DeleteProjectByID(ctx context.Context, id uuid.UUID) (CoreProj
 		&i.Name,
 		&i.Description,
 		&i.CreatedAt,
+		&i.RetentionEventsDays,
+		&i.RetentionAuditDays,
+		&i.RetentionWebhookDays,
 	)
 	return i, err
 }
 
 const getProjectByID = `-- name: GetProjectByID :one
-SELECT id, organization_id, slug, name, description, created_at FROM core.projects
+SELECT id, organization_id, slug, name, description, created_at, retention_events_days, retention_audit_days, retention_webhook_days FROM core.projects
 WHERE id = $1
 `
 
@@ -123,6 +129,9 @@ func (q *Queries) GetProjectByID(ctx context.Context, id uuid.UUID) (CoreProject
 		&i.Name,
 		&i.Description,
 		&i.CreatedAt,
+		&i.RetentionEventsDays,
+		&i.RetentionAuditDays,
+		&i.RetentionWebhookDays,
 	)
 	return i, err
 }
@@ -327,7 +336,7 @@ func (q *Queries) ListProjectMembers(ctx context.Context, projectID uuid.UUID) (
 
 const listProjectsForOrganization = `-- name: ListProjectsForOrganization :many
 SELECT
-  p.id, p.organization_id, p.slug, p.name, p.description, p.created_at,
+  p.id, p.organization_id, p.slug, p.name, p.description, p.created_at, p.retention_events_days, p.retention_audit_days, p.retention_webhook_days,
   pm.role AS membership_role
 FROM core.projects p
 LEFT JOIN core.project_members pm
@@ -350,13 +359,16 @@ type ListProjectsForOrganizationParams struct {
 }
 
 type ListProjectsForOrganizationRow struct {
-	ID             uuid.UUID           `json:"id"`
-	OrganizationID uuid.UUID           `json:"organization_id"`
-	Slug           string              `json:"slug"`
-	Name           string              `json:"name"`
-	Description    *string             `json:"description"`
-	CreatedAt      pgtype.Timestamptz  `json:"created_at"`
-	MembershipRole NullCoreProjectRole `json:"membership_role"`
+	ID                   uuid.UUID           `json:"id"`
+	OrganizationID       uuid.UUID           `json:"organization_id"`
+	Slug                 string              `json:"slug"`
+	Name                 string              `json:"name"`
+	Description          *string             `json:"description"`
+	CreatedAt            pgtype.Timestamptz  `json:"created_at"`
+	RetentionEventsDays  int32               `json:"retention_events_days"`
+	RetentionAuditDays   int32               `json:"retention_audit_days"`
+	RetentionWebhookDays int32               `json:"retention_webhook_days"`
+	MembershipRole       NullCoreProjectRole `json:"membership_role"`
 }
 
 func (q *Queries) ListProjectsForOrganization(ctx context.Context, arg ListProjectsForOrganizationParams) ([]ListProjectsForOrganizationRow, error) {
@@ -375,6 +387,9 @@ func (q *Queries) ListProjectsForOrganization(ctx context.Context, arg ListProje
 			&i.Name,
 			&i.Description,
 			&i.CreatedAt,
+			&i.RetentionEventsDays,
+			&i.RetentionAuditDays,
+			&i.RetentionWebhookDays,
 			&i.MembershipRole,
 		); err != nil {
 			return nil, err
@@ -416,16 +431,22 @@ const updateProjectByID = `-- name: UpdateProjectByID :one
 UPDATE core.projects
 SET slug = $2,
     name = $3,
-    description = $4
+    description = $4,
+    retention_events_days = COALESCE($5, retention_events_days),
+    retention_audit_days = COALESCE($6, retention_audit_days),
+    retention_webhook_days = COALESCE($7, retention_webhook_days)
 WHERE id = $1
-RETURNING id, organization_id, slug, name, description, created_at
+RETURNING id, organization_id, slug, name, description, created_at, retention_events_days, retention_audit_days, retention_webhook_days
 `
 
 type UpdateProjectByIDParams struct {
-	ID          uuid.UUID `json:"id"`
-	Slug        string    `json:"slug"`
-	Name        string    `json:"name"`
-	Description *string   `json:"description"`
+	ID                   uuid.UUID `json:"id"`
+	Slug                 string    `json:"slug"`
+	Name                 string    `json:"name"`
+	Description          *string   `json:"description"`
+	RetentionEventsDays  *int32    `json:"retention_events_days"`
+	RetentionAuditDays   *int32    `json:"retention_audit_days"`
+	RetentionWebhookDays *int32    `json:"retention_webhook_days"`
 }
 
 func (q *Queries) UpdateProjectByID(ctx context.Context, arg UpdateProjectByIDParams) (CoreProject, error) {
@@ -434,6 +455,9 @@ func (q *Queries) UpdateProjectByID(ctx context.Context, arg UpdateProjectByIDPa
 		arg.Slug,
 		arg.Name,
 		arg.Description,
+		arg.RetentionEventsDays,
+		arg.RetentionAuditDays,
+		arg.RetentionWebhookDays,
 	)
 	var i CoreProject
 	err := row.Scan(
@@ -443,6 +467,9 @@ func (q *Queries) UpdateProjectByID(ctx context.Context, arg UpdateProjectByIDPa
 		&i.Name,
 		&i.Description,
 		&i.CreatedAt,
+		&i.RetentionEventsDays,
+		&i.RetentionAuditDays,
+		&i.RetentionWebhookDays,
 	)
 	return i, err
 }

apps/backend/internal/database/db/querier.go

@@ -100,6 +100,15 @@ type Querier interface {
 	RemoveProjectMember(ctx context.Context, arg RemoveProjectMemberParams) (CoreProjectMember, error)
 	RemoveProjectMembershipsForOrganizationUser(ctx context.Context, arg RemoveProjectMembershipsForOrganizationUserParams) error
 	RevokeAPIKey(ctx context.Context, arg RevokeAPIKeyParams) (CoreApiKey, error)
+	SweepExpiredAuditLogs(ctx context.Context) (int64, error)
+	// Per-project retention sweeps. A retention of 0 days disables the sweep for
+	// that stream. Pending webhook deliveries are never swept — they may still be
+	// retrying.
+	SweepExpiredEvents(ctx context.Context) (int64, error)
+	// Org-scoped audit rows (project_id NULL) have no project retention — sweep
+	// them on a fixed 365-day floor so they cannot grow unbounded.
+	SweepExpiredOrgAuditLogs(ctx context.Context) (int64, error)
+	SweepExpiredWebhookDeliveries(ctx context.Context) (int64, error)
 	TouchAPIKey(ctx context.Context, id uuid.UUID) error
 	UpdateBucketByID(ctx context.Context, arg UpdateBucketByIDParams) (CoreBucket, error)
 	UpdateCollection(ctx context.Context, arg UpdateCollectionParams) (CoreCollection, error)

apps/backend/internal/database/db/retention.sql.go

@@ -0,0 +1,80 @@
+// Code generated by sqlc. DO NOT EDIT.
+// versions:
+//   sqlc v1.29.0
+// source: retention.sql
+
+package db
+
+import (
+	"context"
+)
+
+const sweepExpiredAuditLogs = `-- name: SweepExpiredAuditLogs :execrows
+DELETE FROM core.audit_logs a
+USING core.projects p
+WHERE a.project_id = p.id
+  AND p.retention_audit_days > 0
+  AND a.created_at < NOW() - make_interval(days => p.retention_audit_days)
+`
+
+func (q *Queries) SweepExpiredAuditLogs(ctx context.Context) (int64, error) {
+	result, err := q.db.Exec(ctx, sweepExpiredAuditLogs)
+	if err != nil {
+		return 0, err
+	}
+	return result.RowsAffected(), nil
+}
+
+const sweepExpiredEvents = `-- name: SweepExpiredEvents :execrows
+
+DELETE FROM core.events e
+USING core.projects p
+WHERE e.project_id = p.id
+  AND p.retention_events_days > 0
+  AND e.ts < NOW() - make_interval(days => p.retention_events_days)
+`
+
+// Per-project retention sweeps. A retention of 0 days disables the sweep for
+// that stream. Pending webhook deliveries are never swept — they may still be
+// retrying.
+func (q *Queries) SweepExpiredEvents(ctx context.Context) (int64, error) {
+	result, err := q.db.Exec(ctx, sweepExpiredEvents)
+	if err != nil {
+		return 0, err
+	}
+	return result.RowsAffected(), nil
+}
+
+const sweepExpiredOrgAuditLogs = `-- name: SweepExpiredOrgAuditLogs :execrows
+DELETE FROM core.audit_logs a
+WHERE a.project_id IS NULL
+  AND a.created_at < NOW() - INTERVAL '365 days'
+`
+
+// Org-scoped audit rows (project_id NULL) have no project retention — sweep
+// them on a fixed 365-day floor so they cannot grow unbounded.
+func (q *Queries) SweepExpiredOrgAuditLogs(ctx context.Context) (int64, error) {
+	result, err := q.db.Exec(ctx, sweepExpiredOrgAuditLogs)
+	if err != nil {
+		return 0, err
+	}
+	return result.RowsAffected(), nil
+}
+
+const sweepExpiredWebhookDeliveries = `-- name: SweepExpiredWebhookDeliveries :execrows
+DELETE FROM core.webhook_deliveries d
+USING core.webhooks w
+JOIN core.projects p ON p.id = w.project_id
+WHERE d.webhook_id = w.id
+  AND p.retention_webhook_days > 0
+  AND d.status <> 'pending'
+  AND d.created_at < NOW() - make_interval(days => p.retention_webhook_days)
+`
+
+func (q *Queries) SweepExpiredWebhookDeliveries(ctx context.Context) (int64, error) {
+	result, err := q.db.Exec(ctx, sweepExpiredWebhookDeliveries)
+	if err != nil {
+		return 0, err
+	}
+	return result.RowsAffected(), nil
+}

apps/backend/internal/database/db/telemetry.sql.go

@@ -239,6 +239,7 @@ const listComponents = `-- name: ListComponents :many
 SELECT id, project_id, name, kind, meta, created_at FROM core.components
 WHERE project_id = $1
 ORDER BY name
+LIMIT 500
 `
 
 func (q *Queries) ListComponents(ctx context.Context, projectID uuid.UUID) ([]CoreComponent, error) {

apps/backend/internal/handlers/databases.go

@@ -204,6 +204,23 @@ func (h *HTTPHandler) dbxRedis(c *gin.Context) {
 	c.JSON(http.StatusOK, result)
 }
 
+func (h *HTTPHandler) dbxTransfer(c *gin.Context) {
+	actor, projectID, connectionID, ok := h.actorProjectConn(c)
+	if !ok {
+		return
+	}
+	var body services.TransferInput
+	if !h.bindAndValidate(c, &body) {
+		return
+	}
+	result, err := h.Platform.TransferDBRows(c.Request.Context(), actor, projectID, connectionID, body, middleware.RequestIDFromContext(c))
+	if err != nil {
+		h.dbxError(c, err)
+		return
+	}
+	c.JSON(http.StatusOK, result)
+}
+
 // actorAndProject resolves the actor and :projectID param.
 func (h *HTTPHandler) actorAndProject(c *gin.Context) (*models.Actor, uuid.UUID, bool) {
 	actor, ok := middleware.RequireActor(c)

apps/backend/internal/handlers/http.go

@@ -29,6 +29,7 @@ func (h *HTTPHandler) Register(router *gin.Engine) {
 	router.GET("/api/v1/health/liveness", h.liveness)
 	router.GET("/api/v1/health/readiness", h.readiness)
 	router.GET("/api/v1/openapi.yaml", h.openapi)
+	router.GET("/api/v1/metrics", h.metrics)
 
 	api := router.Group("/api/v1")
 	api.GET("/me", h.me)
@@ -91,6 +92,7 @@ func (h *HTTPHandler) Register(router *gin.Engine) {
 	api.GET("/projects/:projectID/db-connections/:connectionID/foreign-keys", h.dbxForeignKeys)
 	api.POST("/projects/:projectID/db-connections/:connectionID/query", h.dbxQuery)
 	api.POST("/projects/:projectID/db-connections/:connectionID/redis", h.dbxRedis)
+	api.POST("/projects/:projectID/db-connections/:connectionID/transfer", h.dbxTransfer)
 	api.GET("/projects/:projectID/integrations", h.listIntegrations)
 	api.POST("/projects/:projectID/integrations", h.createIntegration)
 	api.DELETE("/projects/:projectID/integrations/:integrationID", h.deleteIntegration)
@@ -123,6 +125,12 @@ func (h *HTTPHandler) openapi(c *gin.Context) {
 	c.File("openapi/openapi.yaml")
 }
 
+// metrics is intentionally unauthenticated — Prometheus scrapes it. Restrict
+// it at the edge if the deployment is public-facing.
+func (h *HTTPHandler) metrics(c *gin.Context) {
+	c.Data(http.StatusOK, "text/plain; version=0.0.4; charset=utf-8", []byte(h.Metrics.RenderPrometheus()))
+}
+
 func (h *HTTPHandler) me(c *gin.Context) {
 	actor, ok := middleware.RequireActor(c)
 	if !ok {

apps/backend/internal/handlers/serializers.go

@@ -17,12 +17,15 @@ type organizationMembershipResponse struct {
 }
 
 type projectResponse struct {
-	ID             string  `json:"id"`
-	OrganizationID string  `json:"organization_id"`
-	Slug           string  `json:"slug"`
-	Name           string  `json:"name"`
-	Description    *string `json:"description"`
-	MembershipRole *string `json:"membership_role"`
+	ID                   string  `json:"id"`
+	OrganizationID       string  `json:"organization_id"`
+	Slug                 string  `json:"slug"`
+	Name                 string  `json:"name"`
+	Description          *string `json:"description"`
+	MembershipRole       *string `json:"membership_role"`
+	RetentionEventsDays  int32   `json:"retention_events_days"`
+	RetentionAuditDays   int32   `json:"retention_audit_days"`
+	RetentionWebhookDays int32   `json:"retention_webhook_days"`
 }
 
 type apiKeyResponse struct {
@@ -30,6 +33,7 @@ type apiKeyResponse struct {
 	ProjectID  string     `json:"project_id"`
 	Name       string     `json:"name"`
 	Prefix     string     `json:"prefix"`
+	Scopes     []string   `json:"scopes"`
 	LastUsedAt *time.Time `json:"last_used_at"`
 	RevokedAt  *time.Time `json:"revoked_at"`
 }
@@ -114,23 +118,29 @@ func toProjectFromRow(row db.ListProjectsForOrganizationRow) projectResponse {
 		membershipRole = &role
 	}
 	return projectResponse{
-		ID:             row.ID.String(),
-		OrganizationID: row.OrganizationID.String(),
-		Slug:           row.Slug,
-		Name:           row.Name,
-		Description:    row.Description,
-		MembershipRole: membershipRole,
+		ID:                   row.ID.String(),
+		OrganizationID:       row.OrganizationID.String(),
+		Slug:                 row.Slug,
+		Name:                 row.Name,
+		Description:          row.Description,
+		MembershipRole:       membershipRole,
+		RetentionEventsDays:  row.RetentionEventsDays,
+		RetentionAuditDays:   row.RetentionAuditDays,
+		RetentionWebhookDays: row.RetentionWebhookDays,
 	}
 }
 
 func toProjectFromCore(row db.CoreProject) projectResponse {
 	return projectResponse{
-		ID:             row.ID.String(),
-		OrganizationID: row.OrganizationID.String(),
-		Slug:           row.Slug,
-		Name:           row.Name,
-		Description:    row.Description,
-		MembershipRole: nil,
+		ID:                   row.ID.String(),
+		OrganizationID:       row.OrganizationID.String(),
+		Slug:                 row.Slug,
+		Name:                 row.Name,
+		Description:          row.Description,
+		MembershipRole:       nil,
+		RetentionEventsDays:  row.RetentionEventsDays,
+		RetentionAuditDays:   row.RetentionAuditDays,
+		RetentionWebhookDays: row.RetentionWebhookDays,
 	}
 }
 
@@ -148,6 +158,7 @@ func toAPIKeyResponse(row db.CoreApiKey) apiKeyResponse {
 		ProjectID:  row.ProjectID.String(),
 		Name:       row.Name,
 		Prefix:     row.Prefix,
+		Scopes:     row.Scopes,
 		LastUsedAt: timestamptzPtr(row.LastUsedAt),
 		RevokedAt:  timestamptzPtr(row.RevokedAt),
 	}

apps/backend/internal/middleware/context.go

@@ -125,6 +125,10 @@ func (m AuthMiddleware) ResolveActor() gin.HandlerFunc {
 			if !m.enforceRateLimit(c, "apikey", actor.APIKeyPrefix, m.RateLimits.APIKeyPerMinute, "API key rate limit exceeded") {
 				return
 			}
+			if need := requiredAPIKeyScope(c.Request.Method, c.Request.URL.Path); !actor.HasScope(need) {
+				apperrors.Abort(c, http.StatusForbidden, "insufficient_scope", "api key lacks the required scope: "+need)
+				return
+			}
 			c.Set(actorKey, actor)
 		}
 		c.Next()
@@ -264,9 +268,36 @@ func (m AuthMiddleware) resolveAPIKeyActor(ctx context.Context, rawKey string) (
 		OrganizationID: &orgID,
 		APIKeyID:       &apiKeyID,
 		APIKeyPrefix:   prefix,
+		Scopes:         row.Scopes,
 	}, nil
 }
 
+// requiredAPIKeyScope maps a request to the scope an API key must hold.
+// admin-only surfaces: org management, invitations, bootstrap, project
+// settings/members, and API key minting (a write key must never mint keys).
+func requiredAPIKeyScope(method, path string) string {
+	switch method {
+	case http.MethodGet, http.MethodHead, http.MethodOptions:
+		return "read"
+	}
+	if path == "/api/v1/ingest" {
+		return "ingest"
+	}
+	rest := strings.TrimPrefix(path, "/api/v1/")
+	segs := strings.SplitN(rest, "/", 3)
+	switch segs[0] {
+	case "organizations", "invitations", "bootstrap":
+		return "admin"
+	case "projects":
+		if len(segs) < 3 ||
+			strings.HasPrefix(segs[2], "api-keys") ||
+			strings.HasPrefix(segs[2], "members") {
+			return "admin"
+		}
+	}
+	return "write"
+}
+
 func PgUUID(id uuid.UUID) pgtype.UUID {
 	return pgtype.UUID{Bytes: id, Valid: true}
 }

apps/backend/internal/middleware/context_test.go

@@ -0,0 +1,66 @@
+package middleware
+
+import (
+	"net/http"
+	"testing"
+
+	"github.com/tdvorak/primora/apps/backend/internal/models"
+)
+
+func TestRequiredAPIKeyScope(t *testing.T) {
+	cases := []struct {
+		method, path, want string
+	}{
+		{http.MethodPost, "/api/v1/ingest", "ingest"},
+		{http.MethodOptions, "/api/v1/ingest", "read"}, // preflight passes through CORS
+		{http.MethodGet, "/api/v1/projects/x/events", "read"},
+		{http.MethodGet, "/api/v1/projects/x/api-keys", "read"},
+		{http.MethodPost, "/api/v1/projects/x/api-keys", "admin"},
+		{http.MethodDelete, "/api/v1/projects/x/api-keys/y", "admin"},
+		{http.MethodPatch, "/api/v1/projects/x/members/y", "admin"},
+		{http.MethodPatch, "/api/v1/projects/x", "admin"},
+		{http.MethodDelete, "/api/v1/projects/x", "admin"},
+		{http.MethodPatch, "/api/v1/organizations/x", "admin"},
+		{http.MethodDelete, "/api/v1/organizations/x", "admin"},
+		{http.MethodPost, "/api/v1/organizations/x/invitations", "admin"},
+		{http.MethodPost, "/api/v1/bootstrap", "admin"},
+		{http.MethodPost, "/api/v1/projects/x/buckets", "write"},
+		{http.MethodPost, "/api/v1/projects/x/db-connections/y/transfer", "write"},
+		{http.MethodDelete, "/api/v1/buckets/x", "write"},
+		{http.MethodPost, "/api/v1/projects/x/deploy-markers", "write"},
+	}
+	for _, tc := range cases {
+		if got := requiredAPIKeyScope(tc.method, tc.path); got != tc.want {
+			t.Errorf("%s %s: got %q, want %q", tc.method, tc.path, got, tc.want)
+		}
+	}
+}
+
+func TestActorHasScope(t *testing.T) {
+	key := func(scopes ...string) *models.Actor {
+		return &models.Actor{Type: models.ActorTypeAPIKey, Scopes: scopes}
+	}
+	user := &models.Actor{Type: models.ActorTypeUser}
+
+	if !user.HasScope("admin") {
+		t.Error("user actors bypass scope checks")
+	}
+	if !key("admin").HasScope("read") {
+		t.Error("admin should satisfy every requirement")
+	}
+	if !key("write").HasScope("read") || !key("write").HasScope("ingest") {
+		t.Error("write should imply read and ingest")
+	}
+	if key("write").HasScope("admin") {
+		t.Error("write must not imply admin")
+	}
+	if key("read").HasScope("write") || key("read").HasScope("ingest") {
+		t.Error("read must not imply write or ingest")
+	}
+	if !key("ingest").HasScope("ingest") || key("ingest").HasScope("read") {
+		t.Error("ingest keys only reach /ingest")
+	}
+	if key().HasScope("read") {
+		t.Error("scopeless key has no access")
+	}
+}

apps/backend/internal/models/actor.go

@@ -21,6 +21,7 @@ type Actor struct {
 	OrganizationID *uuid.UUID
 	APIKeyID       *uuid.UUID
 	APIKeyPrefix   string
+	Scopes         []string
 }
 
 func (a *Actor) IsUser() bool {
@@ -30,3 +31,35 @@ func (a *Actor) IsUser() bool {
 func (a *Actor) IsAPIKey() bool {
 	return a != nil && a.Type == ActorTypeAPIKey
 }
+
+// HasScope reports whether the actor may perform work requiring `need`.
+// User actors are always allowed — scopes constrain API keys only.
+// Scope semantics: admin unlocks everything; write unlocks read + ingest +
+// non-admin mutations; read and ingest unlock only themselves.
+func (a *Actor) HasScope(need string) bool {
+	if a == nil {
+		return false
+	}
+	if a.IsUser() {
+		return true
+	}
+	for _, s := range a.Scopes {
+		switch s {
+		case "admin":
+			return true
+		case "write":
+			if need == "read" || need == "write" || need == "ingest" {
+				return true
+			}
+		case "read":
+			if need == "read" {
+				return true
+			}
+		case "ingest":
+			if need == "ingest" {
+				return true
+			}
+		}
+	}
+	return false
+}

apps/backend/internal/observability/metrics.go

@@ -1,6 +1,7 @@
 package observability
 
 import (
+	"fmt"
 	"sync"
 	"time"
 )
@@ -43,12 +44,12 @@ func (m *Metrics) DecrementActive() {
 func (m *Metrics) GetStats() map[string]any {
 	m.mu.RLock()
 	defer m.mu.RUnlock()
-	
+
 	avgResponseTime := int64(0)
 	if m.requestCount > 0 {
 		avgResponseTime = m.totalResponseTime.Milliseconds() / m.requestCount
 	}
-	
+
 	return map[string]any{
 		"total_requests":       m.requestCount,
 		"total_errors":         m.errorCount,
@@ -56,3 +57,24 @@ func (m *Metrics) GetStats() map[string]any {
 		"avg_response_time_ms": avgResponseTime,
 	}
 }
+
+// RenderPrometheus emits the collector in Prometheus text exposition format.
+// Duration is exposed as sum+count so scrapers can compute real averages.
+func (m *Metrics) RenderPrometheus() string {
+	m.mu.RLock()
+	defer m.mu.RUnlock()
+	return fmt.Sprintf(`# HELP primora_http_requests_total Total HTTP requests handled.
+# TYPE primora_http_requests_total counter
+primora_http_requests_total %d
+# HELP primora_http_request_errors_total HTTP requests that ended in a 5xx response.
+# TYPE primora_http_request_errors_total counter
+primora_http_request_errors_total %d
+# HELP primora_http_active_requests Requests currently in flight.
+# TYPE primora_http_active_requests gauge
+primora_http_active_requests %d
+# HELP primora_http_request_duration_ms_sum Sum of request durations in milliseconds.
+# TYPE primora_http_request_duration_ms_sum counter
+primora_http_request_duration_ms_sum %d
+`,
+		m.requestCount, m.errorCount, m.activeRequests, m.totalResponseTime.Milliseconds())
+}

apps/backend/internal/services/databases.go

@@ -500,3 +500,223 @@ func (s *PlatformService) ExecuteDBXRedis(ctx context.Context, actor *models.Act
 	}
 	return map[string]any{"output": text}, nil
 }
+
+// TransferInput describes a DB-to-DB link: rows read from the source
+// connection are written to the target connection.
+type TransferInput struct {
+	TargetConnectionID string `json:"target_connection_id" validate:"required,uuid"`
+	Query              string `json:"query" validate:"required"`
+	Database           string `json:"database"`
+	// SQL targets: destination table (schema-qualified allowed).
+	TargetTable string `json:"target_table"`
+	// Redis targets: SET key template with {column} placeholders; the value is
+	// value_column's cell, or the whole row as JSON when omitted.
+	KeyPattern  string `json:"key_pattern"`
+	ValueColumn string `json:"value_column"`
+	// Optional TTL in seconds applied to each Redis SET.
+	TTLSeconds *int `json:"ttl_seconds" validate:"omitempty,min=0"`
+	Limit      int   `json:"limit" validate:"omitempty,min=1,max=1000"`
+}
+
+const transferMaxRows = 1000
+
+// TransferDBRows moves data between two saved connections of the same
+// project. Both connections must accept write-level access.
+func (s *PlatformService) TransferDBRows(ctx context.Context, actor *models.Actor, projectID, sourceID uuid.UUID, input TransferInput, requestID string) (map[string]any, error) {
+	source, err := s.connForTool(ctx, actor, projectID, sourceID, true)
+	if err != nil {
+		return nil, err
+	}
+	targetID, err := uuid.Parse(input.TargetConnectionID)
+	if err != nil {
+		return nil, errors.New("target_connection_id must be a uuid")
+	}
+	target, err := s.connForTool(ctx, actor, projectID, targetID, true)
+	if err != nil {
+		return nil, err
+	}
+
+	limit := input.Limit
+	if limit <= 0 || limit > transferMaxRows {
+		limit = transferMaxRows
+	}
+	args := map[string]any{"connection_name": dbxConnectionName(projectID, source.Name), "sql": input.Query}
+	if input.Database != "" {
+		args["database"] = input.Database
+	}
+	text, err := s.dbx.Call(ctx, "dbx_execute_query", args)
+	if err != nil {
+		return nil, err
+	}
+	table, _ := dbx.ParseMDTable(text)
+	if table == nil || len(table.Columns) == 0 {
+		return nil, errors.New("source query did not return a result table")
+	}
+	rows := table.Rows
+	truncated := false
+	if len(rows) > limit {
+		rows = rows[:limit]
+		truncated = true
+	}
+
+	var transferred int
+	if target.DbType == "redis" {
+		transferred, err = s.transferToRedis(ctx, projectID, target, table.Columns, rows, input)
+	} else {
+		transferred, err = s.transferToSQL(ctx, projectID, target, table.Columns, rows, input)
+	}
+	if err != nil {
+		return nil, err
+	}
+
+	project, perr := s.repo.Queries().GetProjectByID(ctx, projectID)
+	if perr == nil {
+		_, _ = s.repo.Queries().CreateAuditLog(ctx, newAuditParams(project.OrganizationID, projectID, actor, requestID, "db_connection.transferred", "db_connection", source.ID.String(), map[string]any{
+			"source":      source.Name,
+			"target":      target.Name,
+			"transferred": transferred,
+			"truncated":   truncated,
+		}))
+	}
+	return map[string]any{
+		"transferred": transferred,
+		"truncated":   truncated,
+		"source":      source.Name,
+		"target":      target.Name,
+	}, nil
+}
+
+// transferToSQL INSERTs rows into target_table in batches of 100. Cell values
+// arrive as text from DBX's markdown output; literal "NULL" maps to SQL NULL.
+func (s *PlatformService) transferToSQL(ctx context.Context, projectID uuid.UUID, target db.CoreDbConnection, cols []string, rows [][]string, input TransferInput) (int, error) {
+	tableIdent, ok := safeDottedIdent(input.TargetTable)
+	if !ok {
+		return 0, errors.New("target_table must be a plain identifier or schema.table")
+	}
+	colIdents := make([]string, len(cols))
+	for i, col := range cols {
+		ident, ok := safeIdent(col)
+		if !ok {
+			return 0, fmt.Errorf("source column %q is not a usable identifier", col)
+		}
+		colIdents[i] = `"` + ident + `"`
+	}
+
+	transferred := 0
+	const batch = 100
+	for start := 0; start < len(rows); start += batch {
+		end := min(start+batch, len(rows))
+		var values strings.Builder
+		values.WriteString("INSERT INTO " + tableIdent + " (" + strings.Join(colIdents, ", ") + ") VALUES ")
+		for i, row := range rows[start:end] {
+			if i > 0 {
+				values.WriteString(", ")
+			}
+			values.WriteString("(")
+			for j := range cols {
+				if j > 0 {
+					values.WriteString(", ")
+				}
+				cell := ""
+				if j < len(row) {
+					cell = row[j]
+				}
+				if cell == "NULL" {
+					values.WriteString("NULL")
+				} else {
+					values.WriteString(sqlQuote(cell))
+				}
+			}
+			values.WriteString(")")
+		}
+		if _, err := s.dbx.Call(ctx, "dbx_execute_query", map[string]any{
+			"connection_name": dbxConnectionName(projectID, target.Name),
+			"sql":             values.String(),
+		}); err != nil {
+			return transferred, fmt.Errorf("insert batch at row %d: %w", start, err)
+		}
+		transferred += end - start
+	}
+	return transferred, nil
+}
+
+// transferToRedis SETs one key per row. Placeholders like {email} in
+// key_pattern are replaced with that row's cell value.
+var keyPlaceholder = regexp.MustCompile(`\{([A-Za-z0-9_]+)\}`)
+
+func (s *PlatformService) transferToRedis(ctx context.Context, projectID uuid.UUID, target db.CoreDbConnection, cols []string, rows [][]string, input TransferInput) (int, error) {
+	if input.KeyPattern == "" {
+		return 0, errors.New("key_pattern is required for redis targets")
+	}
+	colIndex := map[string]int{}
+	for i, col := range cols {
+		colIndex[col] = i
+	}
+	for _, ph := range keyPlaceholder.FindAllStringSubmatch(input.KeyPattern, -1) {
+		if _, ok := colIndex[ph[1]]; !ok {
+			return 0, fmt.Errorf("key_pattern references unknown column %q", ph[1])
+		}
+	}
+	if input.ValueColumn != "" {
+		if _, ok := colIndex[input.ValueColumn]; !ok {
+			return 0, fmt.Errorf("value_column %q not in source columns", input.ValueColumn)
+		}
+	}
+
+	transferred := 0
+	for _, row := range rows {
+		key := keyPlaceholder.ReplaceAllStringFunc(input.KeyPattern, func(m string) string {
+			col := keyPlaceholder.FindStringSubmatch(m)[1]
+			if i, ok := colIndex[col]; ok && i < len(row) {
+				return row[i]
+			}
+			return ""
+		})
+		var value string
+		if input.ValueColumn != "" {
+			value = row[colIndex[input.ValueColumn]]
+		} else {
+			obj := map[string]string{}
+			for i, col := range cols {
+				if i < len(row) {
+					obj[col] = row[i]
+				}
+			}
+			raw, _ := json.Marshal(obj)
+			value = string(raw)
+		}
+		cmd := "SET " + redisQuote(key) + " " + redisQuote(value)
+		if input.TTLSeconds != nil && *input.TTLSeconds > 0 {
+			cmd += fmt.Sprintf(" EX %d", *input.TTLSeconds)
+		}
+		if _, err := s.dbx.Call(ctx, "dbx_execute_redis_command", map[string]any{
+			"connection_name": dbxConnectionName(projectID, target.Name),
+			"command":         cmd,
+		}); err != nil {
+			return transferred, fmt.Errorf("SET %q: %w", key, err)
+		}
+		transferred++
+	}
+	return transferred, nil
+}
+
+var dottedIdent = regexp.MustCompile(`^[A-Za-z0-9_]+(\.[A-Za-z0-9_]+)?$`)
+
+func safeDottedIdent(s string) (string, bool) {
+	if !dottedIdent.MatchString(s) {
+		return "", false
+	}
+	parts := strings.Split(s, ".")
+	for i := range parts {
+		parts[i] = `"` + parts[i] + `"`
+	}
+	return strings.Join(parts, "."), true
+}
+
+func sqlQuote(v string) string {
+	return "'" + strings.ReplaceAll(v, "'", "''") + "'"
+}
+
+func redisQuote(v string) string {
+	return `"` + strings.ReplaceAll(strings.ReplaceAll(v, `\`, `\\`), `"`, `\"`) + `"`
+}

apps/backend/internal/services/platform_service.go

@@ -67,6 +67,11 @@ type UpdateProjectInput struct {
 	Name        string  `json:"name" validate:"required,min=2"`
 	Slug        string  `json:"slug" validate:"required,min=2"`
 	Description *string `json:"description"`
+	// Retention windows in days; nil leaves the current value, 0 disables
+	// the sweeper for that stream.
+	RetentionEventsDays  *int32 `json:"retention_events_days" validate:"omitempty,min=0,max=3650"`
+	RetentionAuditDays   *int32 `json:"retention_audit_days" validate:"omitempty,min=0,max=3650"`
+	RetentionWebhookDays *int32 `json:"retention_webhook_days" validate:"omitempty,min=0,max=3650"`
 }
 
 type CreateInvitationInput struct {
@@ -78,7 +83,8 @@ type CreateInvitationInput struct {
 }
 
 type CreateAPIKeyInput struct {
-	Name string `json:"name" validate:"required,min=2"`
+	Name   string   `json:"name" validate:"required,min=2"`
+	Scopes []string `json:"scopes" validate:"omitempty,dive,oneof=ingest read write admin"`
 }
 
 type CreateBucketInput struct {
@@ -736,10 +742,13 @@ func (s *PlatformService) UpdateProject(ctx context.Context, actor *models.Actor
 		return db.CoreProject{}, err
 	}
 	updated, err := s.repo.Queries().UpdateProjectByID(ctx, db.UpdateProjectByIDParams{
-		ID:          projectID,
-		Slug:        normalizeSlug(input.Slug),
-		Name:        strings.TrimSpace(input.Name),
-		Description: input.Description,
+		ID:                   projectID,
+		Slug:                 normalizeSlug(input.Slug),
+		Name:                 strings.TrimSpace(input.Name),
+		Description:          input.Description,
+		RetentionEventsDays:  input.RetentionEventsDays,
+		RetentionAuditDays:   input.RetentionAuditDays,
+		RetentionWebhookDays: input.RetentionWebhookDays,
 	})
 	if err != nil {
 		return db.CoreProject{}, err
@@ -1038,6 +1047,7 @@ func (s *PlatformService) CreateAPIKey(ctx context.Context, actor *models.Actor,
 	if err != nil {
 		return nil, fmt.Errorf("generate api key secret: %w", err)
 	}
+	scopes := normalizeAPIKeyScopes(input.Scopes)
 	prefix := "prm_" + prefixSuffix
 	rawKey := prefix + "_" + keySuffix
 	secretHash := sha256.Sum256([]byte(rawKey))
@@ -1046,6 +1056,7 @@ func (s *PlatformService) CreateAPIKey(ctx context.Context, actor *models.Actor,
 		Name:            strings.TrimSpace(input.Name),
 		Prefix:          prefix,
 		SecretHash:      secretHash[:],
+		Scopes:          scopes,
 		CreatedByUserID: pgtype.UUID{Bytes: *actor.UserID, Valid: true},
 	})
 	if err != nil {
@@ -1062,9 +1073,27 @@ func (s *PlatformService) CreateAPIKey(ctx context.Context, actor *models.Actor,
 		"prefix": row.Prefix,
 		"secret": rawKey,
 		"name":   row.Name,
+		"scopes": row.Scopes,
 	}, nil
 }
 
+// normalizeAPIKeyScopes dedupes the requested scopes. An omitted or empty
+// list means a full-access key — the pre-scopes behaviour.
+func normalizeAPIKeyScopes(input []string) []string {
+	if len(input) == 0 {
+		return []string{"admin"}
+	}
+	seen := map[string]bool{}
+	out := make([]string, 0, len(input))
+	for _, s := range input {
+		if !seen[s] {
+			seen[s] = true
+			out = append(out, s)
+		}
+	}
+	return out
+}
+
 func (s *PlatformService) RevokeAPIKey(ctx context.Context, actor *models.Actor, projectID, apiKeyID uuid.UUID, requestID string) error {
 	if err := s.requireProjectRole(ctx, actor, projectID, "admin"); err != nil {
 		return err

apps/backend/internal/services/retention.go

@@ -0,0 +1,63 @@
+package services
+
+import (
+	"context"
+	"log/slog"
+	"time"
+
+	"github.com/tdvorak/primora/apps/backend/internal/repositories"
+)
+
+// RetentionSweeper deletes telemetry events, audit rows, and webhook
+// deliveries that are past each project's retention window. It runs once
+// shortly after boot and then hourly — a missed sweep is not a problem, the
+// next tick catches it.
+type RetentionSweeper struct {
+	repo     *repositories.CoreRepository
+	logger   *slog.Logger
+	interval time.Duration
+}
+
+func NewRetentionSweeper(repo *repositories.CoreRepository, logger *slog.Logger) *RetentionSweeper {
+	return &RetentionSweeper{repo: repo, logger: logger, interval: time.Hour}
+}
+
+func (s *RetentionSweeper) Run(ctx context.Context) {
+	// jarvis: initial delay — let migrations and first-party seeds settle before sweeping.
+	timer := time.NewTimer(time.Minute)
+	defer timer.Stop()
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-timer.C:
+			s.Sweep(ctx)
+			timer.Reset(s.interval)
+		}
+	}
+}
+
+func (s *RetentionSweeper) Sweep(ctx context.Context) {
+	q := s.repo.Queries()
+	for _, sweep := range []struct {
+		name string
+		fn   func(context.Context) (int64, error)
+	}{
+		{"events", q.SweepExpiredEvents},
+		{"audit_logs", q.SweepExpiredAuditLogs},
+		{"audit_logs_org", q.SweepExpiredOrgAuditLogs},
+		{"webhook_deliveries", q.SweepExpiredWebhookDeliveries},
+	} {
+		rows, err := sweep.fn(ctx)
+		if err != nil {
+			if ctx.Err() != nil {
+				return
+			}
+			s.logger.Warn("retention sweep failed", "stream", sweep.name, "error", err)
+			continue
+		}
+		if rows > 0 {
+			s.logger.Info("retention sweep removed rows", "stream", sweep.name, "rows", rows)
+		}
+	}
+}

apps/backend/openapi/openapi.yaml

@@ -37,6 +37,18 @@ paths:
             application/json:
               schema:
                 $ref: "#/components/schemas/ReadinessStatus"
+  /metrics:
+    get:
+      tags: [Health]
+      operationId: getMetrics
+      summary: Prometheus text exposition of platform HTTP metrics
+      responses:
+        "200":
+          description: Prometheus metrics
+          content:
+            text/plain:
+              schema:
+                type: string
   /me:
     get:
       tags: [Platform]
@@ -1092,6 +1104,34 @@ paths:
           $ref: "#/components/responses/ErrorResponse"
         "502":
           $ref: "#/components/responses/ErrorResponse"
+  /projects/{projectID}/db-connections/{connectionID}/transfer:
+    post:
+      tags: [Databases]
+      operationId: transferDbRows
+      summary: Move query results from this connection to another saved connection
+      security:
+        - bearerAuth: []
+        - apiKeyAuth: []
+      parameters:
+        - $ref: "#/components/parameters/ProjectID"
+        - $ref: "#/components/parameters/ConnectionID"
+      requestBody:
+        required: true
+        content:
+          application/json:
+            schema:
+              $ref: "#/components/schemas/DBTransferRequest"
+      responses:
+        "200":
+          description: Transfer result
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/DBTransferResponse"
+        "404":
+          $ref: "#/components/responses/ErrorResponse"
+        "502":
+          $ref: "#/components/responses/ErrorResponse"
   /ingest:
     post:
       tags: [Telemetry]
@@ -1920,6 +1960,21 @@ components:
         description:
           type: string
           nullable: true
+        retention_events_days:
+          type: integer
+          minimum: 0
+          maximum: 3650
+          description: Telemetry event retention; 0 disables the sweep.
+        retention_audit_days:
+          type: integer
+          minimum: 0
+          maximum: 3650
+          description: Audit log retention; 0 disables the sweep.
+        retention_webhook_days:
+          type: integer
+          minimum: 0
+          maximum: 3650
+          description: Webhook delivery retention; 0 disables the sweep.
       required: [name, slug]
     Project:
       type: object
@@ -1940,6 +1995,12 @@ components:
         membership_role:
           type: string
           nullable: true
+        retention_events_days:
+          type: integer
+        retention_audit_days:
+          type: integer
+        retention_webhook_days:
+          type: integer
       required: [id, organization_id, slug, name]
     ProjectOverview:
       type: object
@@ -2124,6 +2185,15 @@ components:
       properties:
         name:
           type: string
+        scopes:
+          type: array
+          items:
+            type: string
+            enum: [ingest, read, write, admin]
+          description: >-
+            Access scopes for the key. admin unlocks everything; write unlocks
+            read + ingest + non-admin mutations; read and ingest unlock only
+            themselves. Omitted means ["admin"] (full access, legacy behaviour).
       required: [name]
     ApiKey:
       type: object
@@ -2138,6 +2208,11 @@ components:
           type: string
         prefix:
           type: string
+        scopes:
+          type: array
+          items:
+            type: string
+            enum: [ingest, read, write, admin]
         last_used_at:
           type: string
           format: date-time
@@ -2159,6 +2234,11 @@ components:
           type: string
         name:
           type: string
+        scopes:
+          type: array
+          items:
+            type: string
+            enum: [ingest, read, write, admin]
       required: [id, prefix, secret, name]
     CreateBucketRequest:
       type: object
@@ -2591,6 +2671,52 @@ components:
         output:
           type: string
       required: [output]
+    DBTransferRequest:
+      type: object
+      properties:
+        target_connection_id:
+          type: string
+          format: uuid
+        query:
+          type: string
+          description: SELECT run against the source connection.
+        database:
+          type: string
+        target_table:
+          type: string
+          description: Destination table for SQL targets (schema-qualified allowed).
+        key_pattern:
+          type: string
+          description: >-
+            Redis targets: SET key template with {column} placeholders, e.g.
+            "user:{id}".
+        value_column:
+          type: string
+          description: >-
+            Redis targets: column whose cell becomes the value. Omitted stores
+            the whole row as JSON.
+        ttl_seconds:
+          type: integer
+          minimum: 0
+          description: Optional EX TTL applied to each Redis SET.
+        limit:
+          type: integer
+          minimum: 1
+          maximum: 1000
+          default: 1000
+      required: [target_connection_id, query]
+    DBTransferResponse:
+      type: object
+      properties:
+        transferred:
+          type: integer
+        truncated:
+          type: boolean
+        source:
+          type: string
+        target:
+          type: string
+      required: [transferred, truncated, source, target]
     IngestEvent:
       type: object
       properties:

apps/frontend/src/App.tsx

@@ -14,6 +14,7 @@ import {
   UpdateOrganizationRequest,
   UpdateProjectMemberRoleRequest,
   UpdateProjectRequest,
+  CreateApiKeyRequest,
   CopyBucketObjectRequest,
   type Bucket,
   type BucketObject,
@@ -180,7 +181,14 @@ export default function App() {
   });
 
   const [projectInput, setProjectInput] = createSignal({ name: "", slug: "", description: "" });
-  const [projectEditInput, setProjectEditInput] = createSignal({ name: "", slug: "", description: "" });
+  const [projectEditInput, setProjectEditInput] = createSignal({
+    name: "",
+    slug: "",
+    description: "",
+    retentionEventsDays: "",
+    retentionAuditDays: "",
+    retentionWebhookDays: "",
+  });
   const [organizationInput, setOrganizationInput] = createSignal({ name: "", slug: "" });
   const [organizationEditInput, setOrganizationEditInput] = createSignal({ name: "", slug: "" });
   const [invitationInput, setInvitationInput] = createSignal({
@@ -190,6 +198,7 @@ export default function App() {
     projectRole: CreateInvitationRequest.projectRole.DEVELOPER as string,
   });
   const [apiKeyName, setApiKeyName] = createSignal("Frontend key");
+  const [apiKeyScopes, setApiKeyScopes] = createSignal<string[]>(["admin"]);
   const [bucketInput, setBucketInput] = createSignal({
     name: "",
     slug: "",
@@ -258,7 +267,13 @@ export default function App() {
     },
   );
 
-  const availableProjects = createMemo<ProjectSummary[]>(() => {
+  type ProjectListItem = ProjectSummary & {
+    retentionEventsDays?: number;
+    retentionAuditDays?: number;
+    retentionWebhookDays?: number;
+  };
+
+  const availableProjects = createMemo<ProjectListItem[]>(() => {
     const items = organizationProjects();
     if (!items) return activeOrganization()?.projects ?? [];
     return items.map((project) => ({
@@ -267,10 +282,13 @@ export default function App() {
       slug: project.slug,
       description: project.description ?? undefined,
       membershipRole: project.membership_role ?? undefined,
+      retentionEventsDays: project.retention_events_days,
+      retentionAuditDays: project.retention_audit_days,
+      retentionWebhookDays: project.retention_webhook_days,
     }));
   });
 
-  const activeProject = createMemo<ProjectSummary | undefined>(() =>
+  const activeProject = createMemo<ProjectListItem | undefined>(() =>
     availableProjects().find((item) => item.id === selectedProjectID()),
   );
   const canUpdateOrganization = createMemo(() => {
@@ -547,8 +565,15 @@ export default function App() {
     const project = activeProject();
     setProjectEditInput(
       project
-        ? { name: project.name, slug: project.slug, description: project.description ?? "" }
-        : { name: "", slug: "", description: "" },
+        ? {
+            name: project.name,
+            slug: project.slug,
+            description: project.description ?? "",
+            retentionEventsDays: project.retentionEventsDays?.toString() ?? "",
+            retentionAuditDays: project.retentionAuditDays?.toString() ?? "",
+            retentionWebhookDays: project.retentionWebhookDays?.toString() ?? "",
+          }
+        : { name: "", slug: "", description: "", retentionEventsDays: "", retentionAuditDays: "", retentionWebhookDays: "" },
     );
   });
 
@@ -713,6 +738,12 @@ export default function App() {
       setProjectMessage("");
       try {
         const trimmed = projectEditInput().description.trim();
+        const parseRetention = (value: string): number | undefined => {
+          const raw = value.trim();
+          if (raw === "") return undefined;
+          const n = Number(raw);
+          return Number.isFinite(n) ? Math.min(3650, Math.max(0, Math.trunc(n))) : undefined;
+        };
         if (isDemo) {
           await demoService.updateProject({ requestBody: { ...projectEditInput(), description: trimmed || null } });
         } else {
@@ -722,6 +753,9 @@ export default function App() {
               name: projectEditInput().name,
               slug: projectEditInput().slug,
               description: trimmed.length > 0 ? trimmed : null,
+              retention_events_days: parseRetention(projectEditInput().retentionEventsDays),
+              retention_audit_days: parseRetention(projectEditInput().retentionAuditDays),
+              retention_webhook_days: parseRetention(projectEditInput().retentionWebhookDays),
             } as UpdateProjectRequest,
           });
         }
@@ -1001,7 +1035,14 @@ export default function App() {
       try {
         const result = isDemo
           ? await demoService.createApiKey({ requestBody: { name: apiKeyName() } })
-          : await ProjectsService.createApiKey({ projectId: projectID, requestBody: { name: apiKeyName() } });
+          : await ProjectsService.createApiKey({
+              projectId: projectID,
+              requestBody: {
+                name: apiKeyName(),
+                scopes: apiKeyScopes() as CreateApiKeyRequest["scopes"],
+              },
+            });
+        setApiKeyScopes(["admin"]);
         await refetchAPIKeys();
         await refetchAuditLogs();
         await refreshProjectOverviewSnapshot();
@@ -1766,6 +1807,7 @@ export default function App() {
             <SettingsPage
               apiKeys={apiKeys()}
               apiKeyName={apiKeyName()}
+              apiKeyScopes={apiKeyScopes()}
               apiKeySecret={apiKeySecret()}
               apiKeyMessage={apiKeyMessage()}
               apiKeyPending={apiKeyPending()}
@@ -1776,6 +1818,9 @@ export default function App() {
               workspacePending={workspacePending()}
               hasActiveOrganization={!!activeOrganization()}
               onApiKeyNameChange={setApiKeyName}
+              onApiKeyScopeToggle={(scope) =>
+                setApiKeyScopes((c) => (c.includes(scope) ? c.filter((s) => s !== scope) : [...c, scope]))
+              }
               onCreateApiKey={createApiKey}
               onDeleteApiKey={revokeApiKey}
               onOrganizationInputChange={(field, value) => setOrganizationInput((c) => ({ ...c, [field]: value }))}

apps/frontend/src/lib/demo-mode.ts

@@ -531,6 +531,9 @@ class DemoService {
       name: p.name,
       description: p.description ?? null,
       membership_role: p.membershipRole ?? null,
+      retention_events_days: 90,
+      retention_audit_days: 90,
+      retention_webhook_days: 90,
     }));
   }
   async listAuthUsers() { await this.delay(); return authUsers; }

apps/frontend/src/pages/ProjectsPage.tsx

@@ -11,16 +11,22 @@ interface ProjectInput {
   description: string;
 }
 
+interface ProjectEditInput extends ProjectInput {
+  retentionEventsDays: string;
+  retentionAuditDays: string;
+  retentionWebhookDays: string;
+}
+
 interface ProjectsPageProps {
   projects: ProjectSummary[];
   selectedProjectID?: string;
   projectInput: ProjectInput;
-  projectEditInput: ProjectInput;
+  projectEditInput: ProjectEditInput;
   projectMessage: string;
   projectPending: boolean;
   canUpdateProject: boolean;
   onProjectInputChange: (field: keyof ProjectInput, value: string) => void;
-  onProjectEditInputChange: (field: keyof ProjectInput, value: string) => void;
+  onProjectEditInputChange: (field: keyof ProjectEditInput, value: string) => void;
   onCreateProject: (event: SubmitEvent) => void;
   onUpdateProject: (event: SubmitEvent) => void;
   onDeleteProject: () => void;
@@ -252,6 +258,38 @@ export function ProjectsPage(props: ProjectsPageProps) {
             value={props.projectEditInput.description}
             onInput={(e) => props.onProjectEditInputChange("description", e.currentTarget.value)}
           />
+          <div>
+            <span class="label">Data retention (days)</span>
+            <p class="label-hint mb-2">
+              Rows older than this are swept hourly. 0 disables a category. Max 3650.
+            </p>
+            <div class="grid gap-3 sm:grid-cols-3">
+              <Input
+                label="Telemetry events"
+                type="number"
+                min={0}
+                max={3650}
+                value={props.projectEditInput.retentionEventsDays}
+                onInput={(e) => props.onProjectEditInputChange("retentionEventsDays", e.currentTarget.value)}
+              />
+              <Input
+                label="Audit log"
+                type="number"
+                min={0}
+                max={3650}
+                value={props.projectEditInput.retentionAuditDays}
+                onInput={(e) => props.onProjectEditInputChange("retentionAuditDays", e.currentTarget.value)}
+              />
+              <Input
+                label="Webhook deliveries"
+                type="number"
+                min={0}
+                max={3650}
+                value={props.projectEditInput.retentionWebhookDays}
+                onInput={(e) => props.onProjectEditInputChange("retentionWebhookDays", e.currentTarget.value)}
+              />
+            </div>
+          </div>
           <ModalFooter>
             <button type="button" class="btn btn-ghost" onClick={() => setSettingsOpen(false)}>
               Cancel

apps/frontend/src/pages/SettingsPage.tsx

@@ -17,9 +17,17 @@ interface OrgInput {
   slug: string;
 }
 
+const API_KEY_SCOPES: Array<{ value: string; label: string; hint: string }> = [
+  { value: "ingest", label: "ingest", hint: "telemetry ingest only" },
+  { value: "read", label: "read", hint: "read endpoints" },
+  { value: "write", label: "write", hint: "read + ingest + mutations" },
+  { value: "admin", label: "admin", hint: "full access" },
+];
+
 interface SettingsPageProps {
   apiKeys?: ApiKey[];
   apiKeyName: string;
+  apiKeyScopes: string[];
   apiKeySecret: string;
   apiKeyMessage: string;
   apiKeyPending: boolean;
@@ -30,6 +38,7 @@ interface SettingsPageProps {
   workspacePending: boolean;
   hasActiveOrganization: boolean;
   onApiKeyNameChange: (name: string) => void;
+  onApiKeyScopeToggle: (scope: string) => void;
   onCreateApiKey: () => void;
   onDeleteApiKey: (id: string) => void;
   onOrganizationInputChange: (field: keyof OrgInput, value: string) => void;
@@ -112,12 +121,34 @@ export function SettingsPage(props: SettingsPageProps) {
             <button
               class="btn btn-primary"
               onClick={props.onCreateApiKey}
-              disabled={props.apiKeyPending || !props.apiKeyName.trim()}
+              disabled={props.apiKeyPending || !props.apiKeyName.trim() || props.apiKeyScopes.length === 0}
             >
               <IconPlus class="w-4 h-4" />
               Create key
             </button>
           </div>
+          <div class="mt-4">
+            <span class="label">Scopes</span>
+            <div class="flex flex-wrap gap-x-5 gap-y-2 mt-1">
+              <For each={API_KEY_SCOPES}>
+                {(scope) => (
+                  <label class="flex items-center gap-2 text-sm text-text-2">
+                    <input
+                      type="checkbox"
+                      class="accent-[var(--accent)]"
+                      checked={props.apiKeyScopes.includes(scope.value)}
+                      disabled={props.apiKeyScopes.length === 1 && props.apiKeyScopes.includes(scope.value)}
+                      onChange={() => props.onApiKeyScopeToggle(scope.value)}
+                    />
+                    <span>
+                      <code>{scope.label}</code>
+                      <span class="text-text-3"> — {scope.hint}</span>
+                    </span>
+                  </label>
+                )}
+              </For>
+            </div>
+          </div>
         </div>
 
         <div class="card card-flush">
@@ -141,6 +172,7 @@ export function SettingsPage(props: SettingsPageProps) {
                   <tr>
                     <th>Name</th>
                     <th>Prefix</th>
+                    <th>Scopes</th>
                     <th>Last used</th>
                     <th>Status</th>
                     <th style="width:1%" />
@@ -154,6 +186,13 @@ export function SettingsPage(props: SettingsPageProps) {
                         <td>
                           <code>{key.prefix}…</code>
                         </td>
+                        <td>
+                          <div class="flex flex-wrap gap-1">
+                            <For each={key.scopes && key.scopes.length > 0 ? key.scopes : ["admin"]}>
+                              {(scope) => <Badge variant="neutral">{scope}</Badge>}
+                            </For>
+                          </div>
+                        </td>
                         <td class="text-text-3 text-xs">
                           {key.last_used_at ? props.formatDate(key.last_used_at) : "Never"}
                         </td>