feat: phase 6 production hardening + DB-to-DB transfer

Security:
- Graceful shutdown: http.Server + signal.NotifyContext, 10s drain,
  retention sweeper cancelled before pool close
- Scoped API keys (migration 00006): ingest/read/write/admin enforced in
  auth middleware; scope picker + badges in Settings UI; scopeless legacy
  keys act as admin. Read-only keys now get 403 on ingest (verified live)
- nginx: baseline security headers on all responses, CSP scoped to SPA;
  fixed add_header inheritance shadowing
- better-auth rate limit enabled unconditionally (was gated on NODE_ENV,
  which the shipped .env sets to development)
- Mailpit moved to docker-compose.dev.yml; prod compose cannot expose it
- setup.sh generates PRIMORA_ENCRYPTION_KEY

Reliability/ops:
- Retention: per-project retention_{events,audit,webhook}_days
  (migration 00007, 0=off) + hourly RetentionSweeper; edit UI in Projects
- GET /api/v1/metrics Prometheus exposition
- compose: healthchecks + restart policies everywhere, dragonfly pinned
  to v2.0.0, depends_on service_healthy
- backup.sh/restore.sh (pg_dump -Fc + storage volume); restore verified
  against scratch postgres+volume
- .dockerignore: exclude target dirs (build context was 4.7GB)

Correctness/testing:
- DBX transfer endpoint: POST /db-connections/:id/transfer — SQL or Redis
  target, 1000-row bound, server-side creds (verified live pg→dragonfly)
- Pagination audit: ListComponents capped at 500 (auto-registers)
- CI: goose up/reset/up job + compose e2e-smoke job (smoke-e2e.sh
  verified locally: signup, bootstrap, scoped keys, ingest, metrics)
- k6 load sanity script: 6650 reqs, ~217/s, 0 real failures, p95 221ms
  (429s = configured rate limiter working)

Docs: DEPLOYMENT_GUIDE.md, PRODUCTION_READINESS.md; verify script
repaired (stale refs + set -e arithmetic abort). ROADMAP: phase 2 +
phase 6 marked complete.

Authored By: TDvorak <[email protected]>

Generated with [Devin](https://devin.ai)

Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com>

apps/backend/internal/services/databases.go

@@ -500,3 +500,223 @@ func (s *PlatformService) ExecuteDBXRedis(ctx context.Context, actor *models.Act
 	}
 	return map[string]any{"output": text}, nil
 }
+
+// TransferInput describes a DB-to-DB link: rows read from the source
+// connection are written to the target connection.
+type TransferInput struct {
+	TargetConnectionID string `json:"target_connection_id" validate:"required,uuid"`
+	Query              string `json:"query" validate:"required"`
+	Database           string `json:"database"`
+	// SQL targets: destination table (schema-qualified allowed).
+	TargetTable string `json:"target_table"`
+	// Redis targets: SET key template with {column} placeholders; the value is
+	// value_column's cell, or the whole row as JSON when omitted.
+	KeyPattern  string `json:"key_pattern"`
+	ValueColumn string `json:"value_column"`
+	// Optional TTL in seconds applied to each Redis SET.
+	TTLSeconds *int `json:"ttl_seconds" validate:"omitempty,min=0"`
+	Limit      int   `json:"limit" validate:"omitempty,min=1,max=1000"`
+}
+
+const transferMaxRows = 1000
+
+// TransferDBRows moves data between two saved connections of the same
+// project. Both connections must accept write-level access.
+func (s *PlatformService) TransferDBRows(ctx context.Context, actor *models.Actor, projectID, sourceID uuid.UUID, input TransferInput, requestID string) (map[string]any, error) {
+	source, err := s.connForTool(ctx, actor, projectID, sourceID, true)
+	if err != nil {
+		return nil, err
+	}
+	targetID, err := uuid.Parse(input.TargetConnectionID)
+	if err != nil {
+		return nil, errors.New("target_connection_id must be a uuid")
+	}
+	target, err := s.connForTool(ctx, actor, projectID, targetID, true)
+	if err != nil {
+		return nil, err
+	}
+
+	limit := input.Limit
+	if limit <= 0 || limit > transferMaxRows {
+		limit = transferMaxRows
+	}
+	args := map[string]any{"connection_name": dbxConnectionName(projectID, source.Name), "sql": input.Query}
+	if input.Database != "" {
+		args["database"] = input.Database
+	}
+	text, err := s.dbx.Call(ctx, "dbx_execute_query", args)
+	if err != nil {
+		return nil, err
+	}
+	table, _ := dbx.ParseMDTable(text)
+	if table == nil || len(table.Columns) == 0 {
+		return nil, errors.New("source query did not return a result table")
+	}
+	rows := table.Rows
+	truncated := false
+	if len(rows) > limit {
+		rows = rows[:limit]
+		truncated = true
+	}
+
+	var transferred int
+	if target.DbType == "redis" {
+		transferred, err = s.transferToRedis(ctx, projectID, target, table.Columns, rows, input)
+	} else {
+		transferred, err = s.transferToSQL(ctx, projectID, target, table.Columns, rows, input)
+	}
+	if err != nil {
+		return nil, err
+	}
+
+	project, perr := s.repo.Queries().GetProjectByID(ctx, projectID)
+	if perr == nil {
+		_, _ = s.repo.Queries().CreateAuditLog(ctx, newAuditParams(project.OrganizationID, projectID, actor, requestID, "db_connection.transferred", "db_connection", source.ID.String(), map[string]any{
+			"source":      source.Name,
+			"target":      target.Name,
+			"transferred": transferred,
+			"truncated":   truncated,
+		}))
+	}
+	return map[string]any{
+		"transferred": transferred,
+		"truncated":   truncated,
+		"source":      source.Name,
+		"target":      target.Name,
+	}, nil
+}
+
+// transferToSQL INSERTs rows into target_table in batches of 100. Cell values
+// arrive as text from DBX's markdown output; literal "NULL" maps to SQL NULL.
+func (s *PlatformService) transferToSQL(ctx context.Context, projectID uuid.UUID, target db.CoreDbConnection, cols []string, rows [][]string, input TransferInput) (int, error) {
+	tableIdent, ok := safeDottedIdent(input.TargetTable)
+	if !ok {
+		return 0, errors.New("target_table must be a plain identifier or schema.table")
+	}
+	colIdents := make([]string, len(cols))
+	for i, col := range cols {
+		ident, ok := safeIdent(col)
+		if !ok {
+			return 0, fmt.Errorf("source column %q is not a usable identifier", col)
+		}
+		colIdents[i] = `"` + ident + `"`
+	}
+
+	transferred := 0
+	const batch = 100
+	for start := 0; start < len(rows); start += batch {
+		end := min(start+batch, len(rows))
+		var values strings.Builder
+		values.WriteString("INSERT INTO " + tableIdent + " (" + strings.Join(colIdents, ", ") + ") VALUES ")
+		for i, row := range rows[start:end] {
+			if i > 0 {
+				values.WriteString(", ")
+			}
+			values.WriteString("(")
+			for j := range cols {
+				if j > 0 {
+					values.WriteString(", ")
+				}
+				cell := ""
+				if j < len(row) {
+					cell = row[j]
+				}
+				if cell == "NULL" {
+					values.WriteString("NULL")
+				} else {
+					values.WriteString(sqlQuote(cell))
+				}
+			}
+			values.WriteString(")")
+		}
+		if _, err := s.dbx.Call(ctx, "dbx_execute_query", map[string]any{
+			"connection_name": dbxConnectionName(projectID, target.Name),
+			"sql":             values.String(),
+		}); err != nil {
+			return transferred, fmt.Errorf("insert batch at row %d: %w", start, err)
+		}
+		transferred += end - start
+	}
+	return transferred, nil
+}
+
+// transferToRedis SETs one key per row. Placeholders like {email} in
+// key_pattern are replaced with that row's cell value.
+var keyPlaceholder = regexp.MustCompile(`\{([A-Za-z0-9_]+)\}`)
+
+func (s *PlatformService) transferToRedis(ctx context.Context, projectID uuid.UUID, target db.CoreDbConnection, cols []string, rows [][]string, input TransferInput) (int, error) {
+	if input.KeyPattern == "" {
+		return 0, errors.New("key_pattern is required for redis targets")
+	}
+	colIndex := map[string]int{}
+	for i, col := range cols {
+		colIndex[col] = i
+	}
+	for _, ph := range keyPlaceholder.FindAllStringSubmatch(input.KeyPattern, -1) {
+		if _, ok := colIndex[ph[1]]; !ok {
+			return 0, fmt.Errorf("key_pattern references unknown column %q", ph[1])
+		}
+	}
+	if input.ValueColumn != "" {
+		if _, ok := colIndex[input.ValueColumn]; !ok {
+			return 0, fmt.Errorf("value_column %q not in source columns", input.ValueColumn)
+		}
+	}
+
+	transferred := 0
+	for _, row := range rows {
+		key := keyPlaceholder.ReplaceAllStringFunc(input.KeyPattern, func(m string) string {
+			col := keyPlaceholder.FindStringSubmatch(m)[1]
+			if i, ok := colIndex[col]; ok && i < len(row) {
+				return row[i]
+			}
+			return ""
+		})
+		var value string
+		if input.ValueColumn != "" {
+			value = row[colIndex[input.ValueColumn]]
+		} else {
+			obj := map[string]string{}
+			for i, col := range cols {
+				if i < len(row) {
+					obj[col] = row[i]
+				}
+			}
+			raw, _ := json.Marshal(obj)
+			value = string(raw)
+		}
+		cmd := "SET " + redisQuote(key) + " " + redisQuote(value)
+		if input.TTLSeconds != nil && *input.TTLSeconds > 0 {
+			cmd += fmt.Sprintf(" EX %d", *input.TTLSeconds)
+		}
+		if _, err := s.dbx.Call(ctx, "dbx_execute_redis_command", map[string]any{
+			"connection_name": dbxConnectionName(projectID, target.Name),
+			"command":         cmd,
+		}); err != nil {
+			return transferred, fmt.Errorf("SET %q: %w", key, err)
+		}
+		transferred++
+	}
+	return transferred, nil
+}
+
+var dottedIdent = regexp.MustCompile(`^[A-Za-z0-9_]+(\.[A-Za-z0-9_]+)?$`)
+
+func safeDottedIdent(s string) (string, bool) {
+	if !dottedIdent.MatchString(s) {
+		return "", false
+	}
+	parts := strings.Split(s, ".")
+	for i := range parts {
+		parts[i] = `"` + parts[i] + `"`
+	}
+	return strings.Join(parts, "."), true
+}
+
+func sqlQuote(v string) string {
+	return "'" + strings.ReplaceAll(v, "'", "''") + "'"
+}
+
+func redisQuote(v string) string {
+	return `"` + strings.ReplaceAll(strings.ReplaceAll(v, `\`, `\\`), `"`, `\"`) + `"`
+}

apps/backend/internal/services/platform_service.go

@@ -67,6 +67,11 @@ type UpdateProjectInput struct {
 	Name        string  `json:"name" validate:"required,min=2"`
 	Slug        string  `json:"slug" validate:"required,min=2"`
 	Description *string `json:"description"`
+	// Retention windows in days; nil leaves the current value, 0 disables
+	// the sweeper for that stream.
+	RetentionEventsDays  *int32 `json:"retention_events_days" validate:"omitempty,min=0,max=3650"`
+	RetentionAuditDays   *int32 `json:"retention_audit_days" validate:"omitempty,min=0,max=3650"`
+	RetentionWebhookDays *int32 `json:"retention_webhook_days" validate:"omitempty,min=0,max=3650"`
 }
 
 type CreateInvitationInput struct {
@@ -78,7 +83,8 @@ type CreateInvitationInput struct {
 }
 
 type CreateAPIKeyInput struct {
-	Name string `json:"name" validate:"required,min=2"`
+	Name   string   `json:"name" validate:"required,min=2"`
+	Scopes []string `json:"scopes" validate:"omitempty,dive,oneof=ingest read write admin"`
 }
 
 type CreateBucketInput struct {
@@ -736,10 +742,13 @@ func (s *PlatformService) UpdateProject(ctx context.Context, actor *models.Actor
 		return db.CoreProject{}, err
 	}
 	updated, err := s.repo.Queries().UpdateProjectByID(ctx, db.UpdateProjectByIDParams{
-		ID:          projectID,
-		Slug:        normalizeSlug(input.Slug),
-		Name:        strings.TrimSpace(input.Name),
-		Description: input.Description,
+		ID:                   projectID,
+		Slug:                 normalizeSlug(input.Slug),
+		Name:                 strings.TrimSpace(input.Name),
+		Description:          input.Description,
+		RetentionEventsDays:  input.RetentionEventsDays,
+		RetentionAuditDays:   input.RetentionAuditDays,
+		RetentionWebhookDays: input.RetentionWebhookDays,
 	})
 	if err != nil {
 		return db.CoreProject{}, err
@@ -1038,6 +1047,7 @@ func (s *PlatformService) CreateAPIKey(ctx context.Context, actor *models.Actor,
 	if err != nil {
 		return nil, fmt.Errorf("generate api key secret: %w", err)
 	}
+	scopes := normalizeAPIKeyScopes(input.Scopes)
 	prefix := "prm_" + prefixSuffix
 	rawKey := prefix + "_" + keySuffix
 	secretHash := sha256.Sum256([]byte(rawKey))
@@ -1046,6 +1056,7 @@ func (s *PlatformService) CreateAPIKey(ctx context.Context, actor *models.Actor,
 		Name:            strings.TrimSpace(input.Name),
 		Prefix:          prefix,
 		SecretHash:      secretHash[:],
+		Scopes:          scopes,
 		CreatedByUserID: pgtype.UUID{Bytes: *actor.UserID, Valid: true},
 	})
 	if err != nil {
@@ -1062,9 +1073,27 @@ func (s *PlatformService) CreateAPIKey(ctx context.Context, actor *models.Actor,
 		"prefix": row.Prefix,
 		"secret": rawKey,
 		"name":   row.Name,
+		"scopes": row.Scopes,
 	}, nil
 }
 
+// normalizeAPIKeyScopes dedupes the requested scopes. An omitted or empty
+// list means a full-access key — the pre-scopes behaviour.
+func normalizeAPIKeyScopes(input []string) []string {
+	if len(input) == 0 {
+		return []string{"admin"}
+	}
+	seen := map[string]bool{}
+	out := make([]string, 0, len(input))
+	for _, s := range input {
+		if !seen[s] {
+			seen[s] = true
+			out = append(out, s)
+		}
+	}
+	return out
+}
+
 func (s *PlatformService) RevokeAPIKey(ctx context.Context, actor *models.Actor, projectID, apiKeyID uuid.UUID, requestID string) error {
 	if err := s.requireProjectRole(ctx, actor, projectID, "admin"); err != nil {
 		return err

apps/backend/internal/services/retention.go

@@ -0,0 +1,63 @@
+package services
+
+import (
+	"context"
+	"log/slog"
+	"time"
+
+	"github.com/tdvorak/primora/apps/backend/internal/repositories"
+)
+
+// RetentionSweeper deletes telemetry events, audit rows, and webhook
+// deliveries that are past each project's retention window. It runs once
+// shortly after boot and then hourly — a missed sweep is not a problem, the
+// next tick catches it.
+type RetentionSweeper struct {
+	repo     *repositories.CoreRepository
+	logger   *slog.Logger
+	interval time.Duration
+}
+
+func NewRetentionSweeper(repo *repositories.CoreRepository, logger *slog.Logger) *RetentionSweeper {
+	return &RetentionSweeper{repo: repo, logger: logger, interval: time.Hour}
+}
+
+func (s *RetentionSweeper) Run(ctx context.Context) {
+	// jarvis: initial delay — let migrations and first-party seeds settle before sweeping.
+	timer := time.NewTimer(time.Minute)
+	defer timer.Stop()
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-timer.C:
+			s.Sweep(ctx)
+			timer.Reset(s.interval)
+		}
+	}
+}
+
+func (s *RetentionSweeper) Sweep(ctx context.Context) {
+	q := s.repo.Queries()
+	for _, sweep := range []struct {
+		name string
+		fn   func(context.Context) (int64, error)
+	}{
+		{"events", q.SweepExpiredEvents},
+		{"audit_logs", q.SweepExpiredAuditLogs},
+		{"audit_logs_org", q.SweepExpiredOrgAuditLogs},
+		{"webhook_deliveries", q.SweepExpiredWebhookDeliveries},
+	} {
+		rows, err := sweep.fn(ctx)
+		if err != nil {
+			if ctx.Err() != nil {
+				return
+			}
+			s.logger.Warn("retention sweep failed", "stream", sweep.name, "error", err)
+			continue
+		}
+		if rows > 0 {
+			s.logger.Info("retention sweep removed rows", "stream", sweep.name, "rows", rows)
+		}
+	}
+}