From 55edbf8b566e3d9c1d9c2ed28ac00b424cc2cd75 Mon Sep 17 00:00:00 2001 From: Tomas Dvorak Date: Thu, 17 Sep 2026 09:15:42 +0200 Subject: [PATCH] feat: phase 6 production hardening + DB-to-DB transfer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Security: - Graceful shutdown: http.Server + signal.NotifyContext, 10s drain, retention sweeper cancelled before pool close - Scoped API keys (migration 00006): ingest/read/write/admin enforced in auth middleware; scope picker + badges in Settings UI; scopeless legacy keys act as admin. Read-only keys now get 403 on ingest (verified live) - nginx: baseline security headers on all responses, CSP scoped to SPA; fixed add_header inheritance shadowing - better-auth rate limit enabled unconditionally (was gated on NODE_ENV, which the shipped .env sets to development) - Mailpit moved to docker-compose.dev.yml; prod compose cannot expose it - setup.sh generates PRIMORA_ENCRYPTION_KEY Reliability/ops: - Retention: per-project retention_{events,audit,webhook}_days (migration 00007, 0=off) + hourly RetentionSweeper; edit UI in Projects - GET /api/v1/metrics Prometheus exposition - compose: healthchecks + restart policies everywhere, dragonfly pinned to v2.0.0, depends_on service_healthy - backup.sh/restore.sh (pg_dump -Fc + storage volume); restore verified against scratch postgres+volume - .dockerignore: exclude target dirs (build context was 4.7GB) Correctness/testing: - DBX transfer endpoint: POST /db-connections/:id/transfer — SQL or Redis target, 1000-row bound, server-side creds (verified live pg→dragonfly) - Pagination audit: ListComponents capped at 500 (auto-registers) - CI: goose up/reset/up job + compose e2e-smoke job (smoke-e2e.sh verified locally: signup, bootstrap, scoped keys, ingest, metrics) - k6 load sanity script: 6650 reqs, ~217/s, 0 real failures, p95 221ms (429s = configured rate limiter working) Docs: DEPLOYMENT_GUIDE.md, PRODUCTION_READINESS.md; verify script repaired (stale refs + set -e arithmetic abort). ROADMAP: phase 2 + phase 6 marked complete. Authored By: TDvorak Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .dockerignore | 9 + .github/workflows/ci.yml | 80 +++++++ .github/workflows/release.yml | 2 + .gitignore | 2 + DEPLOYMENT_GUIDE.md | 151 ++++++++++++ PRODUCTION_READINESS.md | 85 +++++++ ROADMAP.md | 116 ++++----- apps/auth/src/lib/auth.ts | 6 + .../db/migrations/00006_api_key_scopes.sql | 6 + .../backend/db/migrations/00007_retention.sql | 11 + apps/backend/db/queries/api_keys.sql | 3 +- apps/backend/db/queries/projects.sql | 5 +- apps/backend/db/queries/retention.sql | 33 +++ apps/backend/db/queries/telemetry.sql | 3 +- apps/backend/internal/app/app.go | 59 ++++- .../internal/database/db/api_keys.sql.go | 21 +- .../internal/database/db/bootstrap.sql.go | 2 +- apps/backend/internal/database/db/models.go | 16 +- .../internal/database/db/projects.sql.go | 61 +++-- apps/backend/internal/database/db/querier.go | 9 + .../internal/database/db/retention.sql.go | 80 +++++++ .../internal/database/db/telemetry.sql.go | 1 + apps/backend/internal/handlers/databases.go | 17 ++ apps/backend/internal/handlers/http.go | 8 + apps/backend/internal/handlers/serializers.go | 47 ++-- apps/backend/internal/middleware/context.go | 31 +++ .../internal/middleware/context_test.go | 66 ++++++ apps/backend/internal/models/actor.go | 33 +++ .../backend/internal/observability/metrics.go | 26 ++- apps/backend/internal/services/databases.go | 220 ++++++++++++++++++ .../internal/services/platform_service.go | 39 +++- apps/backend/internal/services/retention.go | 63 +++++ apps/backend/openapi/openapi.yaml | 126 ++++++++++ apps/frontend/src/App.tsx | 57 ++++- apps/frontend/src/lib/demo-mode.ts | 3 + apps/frontend/src/pages/ProjectsPage.tsx | 42 +++- apps/frontend/src/pages/SettingsPage.tsx | 41 +++- docker-compose.dev.yml | 28 +++ docker-compose.yml | 50 ++-- infra/nginx/default.conf | 31 ++- infra/nginx/mailpit.conf | 12 + packages/api-client/src/generated/index.ts | 4 + .../api-client/src/generated/models/ApiKey.ts | 1 + .../generated/models/CreateApiKeyRequest.ts | 4 + .../src/generated/models/CreatedApiKey.ts | 1 + .../src/generated/models/DBTransferRequest.ts | 30 +++ .../generated/models/DBTransferResponse.ts | 11 + .../src/generated/models/Project.ts | 3 + .../generated/models/UpdateProjectRequest.ts | 12 + .../src/generated/schemas/$ApiKey.ts | 6 + .../generated/schemas/$CreateApiKeyRequest.ts | 6 + .../src/generated/schemas/$CreatedApiKey.ts | 6 + .../generated/schemas/$DBTransferRequest.ts | 42 ++++ .../generated/schemas/$DBTransferResponse.ts | 24 ++ .../src/generated/schemas/$Project.ts | 9 + .../schemas/$UpdateProjectRequest.ts | 15 ++ .../generated/services/DatabasesService.ts | 31 +++ .../src/generated/services/HealthService.ts | 11 + scripts/backup.sh | 50 ++++ scripts/load-sanity.js | 75 ++++++ scripts/restore.sh | 47 ++++ scripts/setup.sh | 10 +- scripts/smoke-e2e.sh | 102 ++++++++ scripts/verify-production-ready.sh | 22 +- 64 files changed, 2058 insertions(+), 165 deletions(-) create mode 100644 DEPLOYMENT_GUIDE.md create mode 100644 PRODUCTION_READINESS.md create mode 100644 apps/backend/db/migrations/00006_api_key_scopes.sql create mode 100644 apps/backend/db/migrations/00007_retention.sql create mode 100644 apps/backend/db/queries/retention.sql create mode 100644 apps/backend/internal/database/db/retention.sql.go create mode 100644 apps/backend/internal/middleware/context_test.go create mode 100644 apps/backend/internal/services/retention.go create mode 100644 docker-compose.dev.yml create mode 100644 infra/nginx/mailpit.conf create mode 100644 packages/api-client/src/generated/models/DBTransferRequest.ts create mode 100644 packages/api-client/src/generated/models/DBTransferResponse.ts create mode 100644 packages/api-client/src/generated/schemas/$DBTransferRequest.ts create mode 100644 packages/api-client/src/generated/schemas/$DBTransferResponse.ts create mode 100755 scripts/backup.sh create mode 100644 scripts/load-sanity.js create mode 100755 scripts/restore.sh create mode 100755 scripts/smoke-e2e.sh diff --git a/.dockerignore b/.dockerignore index bf7bcdf..a397d18 100644 --- a/.dockerignore +++ b/.dockerignore @@ -7,3 +7,12 @@ coverage .env npm-debug.log +# build artifacts + local data — 4.5G+ of Rust target dir otherwise +**/target +apps/desktop/src-tauri/gen +backups +tmp +**/tmp +graphify-out +.playwright-mcp + diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a8fbf7e..a14aab8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -129,6 +129,86 @@ jobs: push: false tags: primora-frontend:test + migrations: + name: Migration Up/Down + runs-on: ubuntu-latest + + services: + postgres: + image: postgres:17-alpine + env: + POSTGRES_USER: primora_test + POSTGRES_PASSWORD: test + POSTGRES_DB: primora_test + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + ports: + - 5432:5432 + + steps: + - uses: actions/checkout@v4 + + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version: '1.26' + cache-dependency-path: apps/backend/go.sum + + - name: Install goose + run: go install github.com/pressly/goose/v3/cmd/goose@v3.26.0 + + - name: Migrations up + run: | + goose -dir apps/backend/db/migrations \ + postgres "postgres://primora_test:test@localhost:5432/primora_test?sslmode=disable" up + + - name: Migrations reset (all downs) + run: | + goose -dir apps/backend/db/migrations \ + postgres "postgres://primora_test:test@localhost:5432/primora_test?sslmode=disable" reset + + - name: Migrations up again (idempotent re-apply) + run: | + goose -dir apps/backend/db/migrations \ + postgres "postgres://primora_test:test@localhost:5432/primora_test?sslmode=disable" up + + e2e-smoke: + name: Compose E2E Smoke + runs-on: ubuntu-latest + timeout-minutes: 15 + + steps: + - uses: actions/checkout@v4 + + - name: Prepare env + run: cp .env.example .env + + - name: Start stack + run: >- + docker compose + -f docker-compose.yml -f docker-compose.dev.yml + up -d --build + + - name: Run smoke checks + run: ./scripts/smoke-e2e.sh + + - name: Dump logs on failure + if: failure() + run: >- + docker compose + -f docker-compose.yml -f docker-compose.dev.yml + logs --tail=200 + + - name: Tear down + if: always() + run: >- + docker compose + -f docker-compose.yml -f docker-compose.dev.yml + down -v + lint: name: Lint runs-on: ubuntu-latest diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 54aed78..8d07679 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -37,6 +37,7 @@ jobs: app: [backend, auth, frontend] steps: - uses: actions/checkout@v4 + - uses: docker/setup-qemu-action@v3 - uses: docker/setup-buildx-action@v3 - uses: docker/login-action@v3 with: @@ -48,6 +49,7 @@ jobs: with: context: . file: apps/${{ matrix.app }}/Dockerfile + platforms: linux/amd64,linux/arm64 push: true tags: | ghcr.io/${{ github.repository_owner }}/primora-${{ matrix.app }}:${{ env.VERSION }} diff --git a/.gitignore b/.gitignore index 4b81b82..ce8bb0f 100644 --- a/.gitignore +++ b/.gitignore @@ -29,6 +29,8 @@ apps/desktop/src-tauri/gen/schemas # Runtime data infra/data +backups +.restore-tmp.* infra/postgres infra/dragonfly infra/mailpit diff --git a/DEPLOYMENT_GUIDE.md b/DEPLOYMENT_GUIDE.md new file mode 100644 index 0000000..e5387df --- /dev/null +++ b/DEPLOYMENT_GUIDE.md @@ -0,0 +1,151 @@ +# Primora Deployment Guide + +Primora ships as a Docker Compose stack: nginx (edge), frontend (static SPA), +auth (Better Auth / Hono), backend (Go API), postgres, and dragonfly +(Redis-compatible cache). The base `docker-compose.yml` is the production +surface — dev-only services like Mailpit live in `docker-compose.dev.yml`. + +## Prerequisites + +- Docker Engine 24+ with the Compose plugin +- A host with 2 GB free RAM minimum (postgres + dragonfly + services) +- A domain pointing at the host if you want TLS + +## Quick start + +```bash +git clone && cd Primora +./scripts/setup.sh # writes .env with generated secrets +docker compose up -d # production surface only +``` + +For local development (includes Mailpit at `/mailpit/`): + +```bash +docker compose -f docker-compose.yml -f docker-compose.dev.yml up -d +``` + +`setup.sh` generates `JWT_SECRET`, `BETTER_AUTH_SECRET`, and +`PRIMORA_ENCRYPTION_KEY`. Do not reuse the example values anywhere public. + +## Environment reference + +| Variable | Purpose | Required | +|---|---|---| +| `POSTGRES_USER` / `POSTGRES_PASSWORD` / `POSTGRES_DB` | database credentials | yes | +| `DATABASE_URL` | backend → postgres DSN | yes | +| `DRAGONFLY_URL` | backend → cache DSN | yes | +| `JWT_SECRET` | signs API JWTs (`JWT_ISSUER`, `JWT_AUDIENCE`, `JWT_TTL_SECONDS` tune tokens) | yes | +| `BETTER_AUTH_SECRET` | auth session/signing secret | yes | +| `PRIMORA_ENCRYPTION_KEY` | AES-256 key (64 hex chars) for integration credentials + webhook secrets at rest | production | +| `AUTH_BASE_URL` / `BETTER_AUTH_URL` | public auth URL (`https://host/auth`) | yes | +| `AUTH_INTERNAL_BASE_URL` | backend → auth internal URL (`http://auth:3001`) | yes | +| `VITE_APP_URL` / `VITE_AUTH_BASE_URL` / `VITE_API_BASE_URL` | frontend public URLs — **baked into the frontend image at build time** | yes | +| `BACKEND_STORAGE_ROOT` | object storage directory inside the backend container | yes | +| `AUTH_ADMIN_EMAILS` | comma-separated emails promoted to auth admin on boot | no | +| `USER_RATE_LIMIT_PER_MINUTE` / `API_KEY_RATE_LIMIT_PER_MINUTE` | per-identity API rate limits (defaults 240 / 600) | no | +| `SMTP_HOST` / `SMTP_PORT` / `SMTP_USER` / `SMTP_PASSWORD` / `SMTP_SECURE` / `MAIL_FROM` | transactional mail; without it verification/reset emails go nowhere | production | +| `RESEND_API_KEY` | alternative to SMTP — Resend API | no | +| `*_CLIENT_ID` / `*_CLIENT_SECRET` | GitHub / Google / Discord / Microsoft OAuth | no | +| `NGINX_PORT` | host port for the edge (default 80) | no | + +## TLS + +Primora does not terminate TLS itself — put a reverse proxy in front of +nginx's port 80. The nginx config sets `X-Forwarded-*`-aware headers; pass +them through. + +**Caddy** (`Caddyfile`): + +``` +primora.example.com { + reverse_proxy localhost:80 +} +``` + +**Traefik** (compose labels on the nginx service): + +```yaml +labels: + - traefik.enable=true + - traefik.http.routers.primora.rule=Host(`primora.example.com`) + - traefik.http.routers.primora.entrypoints=websecure + - traefik.http.routers.primora.tls.certresolver=letsencrypt +``` + +Once TLS is live, update `VITE_APP_URL`, `VITE_AUTH_BASE_URL`, +`AUTH_BASE_URL`, `BETTER_AUTH_URL`, and `COOKIE_DOMAIN` to the https origin +and rebuild the frontend image (`docker compose build frontend`). + +## SMTP / email + +Auth sends verification and password-reset email through SMTP +(`SMTP_*` + `MAIL_FROM`) or Resend (`RESEND_API_KEY`). In development the +`docker-compose.dev.yml` overlay runs Mailpit; mail is captured at +`http://localhost/mailpit/` and nothing leaves the host. **The production +compose file contains no mail sink — configure real SMTP before inviting +users.** + +## Backups + +`scripts/backup.sh` produces a tarball with a `pg_dump -Fc` of the database +and a tar of the `backend_storage` volume: + +```bash +./scripts/backup.sh # → backups/primora-.tar.gz +``` + +Restore (destructive — stops writes): + +```bash +docker compose stop backend auth +./scripts/restore.sh backups/primora-.tar.gz +docker compose up -d +``` + +Cron example (nightly, keep 14): + +```cron +17 3 * * * cd /opt/primora && ./scripts/backup.sh && \ + ls -1t backups/primora-*.tar.gz | tail -n +15 | xargs -r rm +``` + +systemd timer example is in `PRODUCTION_READINESS.md`. + +## Upgrades + +Migrations run automatically on backend boot (goose). Safe order: + +```bash +./scripts/backup.sh # 1. snapshot first +docker compose pull && docker compose build # 2. new images +docker compose up -d # 3. rolling recreate; backend migrates on boot +./scripts/smoke-e2e.sh # 4. verify health, auth, ingest +``` + +`goose` migrations include `Down` sections; `goose -dir apps/backend/db/migrations postgres "$DATABASE_URL" down` rolls back one version if a release misbehaves. + +## Storage constraint + +Object storage is the **local filesystem** (`backend_storage` volume). There +is no S3-compatible backend yet — the volume must be on durable, backed-up +disk, and a single host serves all objects. Plan capacity accordingly; +multi-node deployments need an external object store first. + +## Observability + +- `GET /api/v1/metrics` — Prometheus exposition (unauthenticated; restrict at + the edge if the instance is public — see the commented rule in + `infra/nginx/default.conf`). +- `GET /api/v1/health/liveness` / `readiness` — compose healthchecks use the + same endpoints. +- Structured JSON logs on stdout for every service — ship with your normal + docker log driver. + +## API key scopes + +Keys are created with scopes: `ingest`, `read`, `write`, `admin`. +`admin` unlocks everything; `write` unlocks read + ingest + non-admin +mutations; `read`/`ingest` are single-purpose. Keys without stored scopes +(pre-scope keys) behave as `admin` for backwards compatibility — rotate them +into scoped keys at your leisure. diff --git a/PRODUCTION_READINESS.md b/PRODUCTION_READINESS.md new file mode 100644 index 0000000..0931860 --- /dev/null +++ b/PRODUCTION_READINESS.md @@ -0,0 +1,85 @@ +# Primora Production Readiness + +Status checklist for self-hosted production use. Everything listed under +"Hardened" is implemented and verified; "Known limits" is the honest +remainder. + +## Hardened + +- **Graceful shutdown** — the backend drains in-flight requests for up to + 10 s on SIGTERM/SIGINT (`internal/app`). +- **Scoped API keys** — `ingest` / `read` / `write` / `admin`, enforced in + auth middleware before handlers run. Legacy keys without stored scopes act + as `admin`. +- **Brute-force protection** — Better Auth rate limiting is enabled + unconditionally (3 attempts / 10 s on sign-in, sign-up, password change); + an IP-level limiter caps `/auth/*` at 60 req/min, and per-identity API + rate limits apply via `USER_RATE_LIMIT_PER_MINUTE` / + `API_KEY_RATE_LIMIT_PER_MINUTE` in Dragonfly. +- **Metrics** — `GET /api/v1/metrics` in Prometheus text format. +- **Retention** — per-project sweeper for telemetry events, audit log rows, + and webhook deliveries (`retention_*_days`, 0 = disabled, default 90). +- **Edge security** — nginx emits `X-Frame-Options`, `X-Content-Type-Options`, + `Referrer-Policy`, `Permissions-Policy`, and a CSP scoped to the SPA only. +- **Compose hygiene** — all services have healthchecks and + `restart: unless-stopped`; Dragonfly is pinned to `v2.0.0`; Mailpit lives + only in `docker-compose.dev.yml`. +- **Secrets** — `setup.sh` generates `JWT_SECRET`, `BETTER_AUTH_SECRET`, and + `PRIMORA_ENCRYPTION_KEY`; credentials and webhook secrets are AES-256 + encrypted at rest. +- **Release images** — GitHub release builds `linux/amd64` + `linux/arm64` + via buildx/QEMU. +- **CI coverage** — Go tests, frontend tests + typecheck, all three Docker + builds, golangci-lint, migration up/reset/up round-trip, and a compose E2E + smoke (`scripts/smoke-e2e.sh`). +- **Backups** — `scripts/backup.sh` / `scripts/restore.sh` cover the database + (custom-format dump) and the storage volume. + +## Known limits + +- **Storage is local filesystem only.** No S3 backend — the + `backend_storage` volume is the durability boundary. +- **Single host.** Compose topology is single-node; postgres and dragonfly + are not clustered. HA is out of scope for v0.x. +- **No TLS inside the stack.** Terminate at a front proxy (Caddy/Traefik); + see DEPLOYMENT_GUIDE.md. +- **Metrics are in-memory counters.** `/api/v1/metrics` covers request + totals/errors/duration — not per-route histograms or DB pool stats yet. +- **`/api/v1/metrics` is unauthenticated.** Scrape-only design; block it at + the edge for public instances (commented rule in `infra/nginx/default.conf`). + +## systemd timer for backups + +`/etc/systemd/system/primora-backup.service`: + +```ini +[Unit] +Description=Primora backup + +[Service] +Type=oneshot +WorkingDirectory=/opt/primora +ExecStart=/opt/primora/scripts/backup.sh +``` + +`/etc/systemd/system/primora-backup.timer`: + +```ini +[Unit] +Description=Nightly Primora backup + +[Timer] +OnCalendar=*-*-* 03:17:00 +Persistent=true + +[Install] +WantedBy=timers.target +``` + +## Verification + +```bash +./scripts/verify-production-ready.sh # structural check +./scripts/smoke-e2e.sh # live E2E against a running stack +npm run check # typecheck + tests + generated-code drift +``` diff --git a/ROADMAP.md b/ROADMAP.md index 20f03f2..ace8a88 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -39,7 +39,7 @@ Everything an operator touches that isn't in the box yet. - **Responsive pass** — sidebar already collapses; audit every page at 390px for table/card overflow. -## Phase 2 — Databases (DBX) +## Phase 2 — Databases (DBX) ✅ The "any database" story, via [DBX](https://github.com/t8y2/dbx) — 90+ drivers in a 25 MB MCP server. Primora does not embed DBX; the Go backend keeps @@ -58,9 +58,10 @@ as REST (pattern already proven in IMS's `internal/app/dbx.go`). - **First-party connections** ✅ — the platform's own Postgres and Dragonfly auto-register as connections on every project, so "put Postgres in it and it runs Postgres" works out of the box. -- **DB-to-DB links** — DBX data transfer between two saved connections - (e.g. Postgres → Dragonfly cache warm), plus documented patterns for - pairing them. +- **DB-to-DB links** ✅ — `POST …/db-connections/:id/transfer` runs a query + on the source connection and writes rows to a SQL target (`target_table`) + or a Redis target (`key_pattern` + `value_column`, optional TTL). Bounded + at 1000 rows; credentials stay server-side. ## Phase 3 — Observability (IMS merge) ✅ @@ -125,7 +126,7 @@ Follow-ups: - Desktop bundle signing + auto-updater — unsigned artifacts today; add when distribution matures. -## Phase 6 — Production hardening (v0.6.0) +## Phase 6 — Production hardening (v0.6.0) ✅ The v1 feature surface is essentially complete; what remains is the difference between "works" and "safe to run other people's data". Roughly @@ -140,74 +141,79 @@ release pipeline with images + binaries + desktop bundles. ### Security -- **Graceful shutdown** — `router.Run()` drops in-flight requests on every - deploy. Switch to `http.Server` + `signal.NotifyContext` with a ~10s - drain. -- **Security headers** — add `X-Frame-Options`, `Referrer-Policy`, - `Permissions-Policy`, and a baseline CSP at the nginx/frontend layer. -- **TLS story** — shipped nginx vhost is HTTP-only. Document a reverse - proxy (Caddy/Traefik) in front for production; optionally ship a - TLS-ready vhost with cert volume mounts. Don't embed a cert manager. -- **Scoped API keys** — `prm_…` keys are full-project-access today. Add a - `scopes` column (`ingest`, `read`, `write`, `admin`) and enforce in the - auth middleware. -- **Dev-surface isolation** — mailpit is proxied at `/mailpit/` in the - single compose file. Move dev services to `docker-compose.dev.yml` so a - production deploy cannot expose it. -- **Auth brute-force** — verify Better Auth's built-in throttling covers - login/signup; add IP backoff if it doesn't. -- **Setup ergonomics** — `setup.sh` should generate - `PRIMORA_ENCRYPTION_KEY` (`openssl rand -hex 32`) rather than leaving it - to the operator. +- ~~**Graceful shutdown**~~ ✅ — `http.Server` + `signal.NotifyContext`, + 10 s drain; verified via SIGTERM on the running container. +- ~~**Security headers**~~ ✅ — `X-Frame-Options`, `X-Content-Type-Options`, + `Referrer-Policy`, `Permissions-Policy` on every response; CSP scoped to + the SPA location only (verified live). +- ~~**TLS story**~~ ✅ — documented Caddy/Traefik front-proxy setups in + DEPLOYMENT_GUIDE.md; no embedded cert manager. +- ~~**Scoped API keys**~~ ✅ — `scopes` column (migration 00006), enforced + in auth middleware (`requiredAPIKeyScope`), UI checkboxes in Settings; + keys without stored scopes behave as `admin` (legacy). Verified live: a + `read`-only key gets 403 on ingest. +- ~~**Dev-surface isolation**~~ ✅ — Mailpit moved to + `docker-compose.dev.yml`; the base compose file cannot expose it. +- ~~**Auth brute-force**~~ ✅ — Better Auth's built-in rate limiting is now + enabled unconditionally (it previously followed `NODE_ENV`, which the + shipped `.env` sets to development); the IP-level 60 req/min limiter on + `/auth/*` stays in front. +- ~~**Setup ergonomics**~~ ✅ — `setup.sh` generates + `PRIMORA_ENCRYPTION_KEY` via `openssl rand -hex 32` alongside the other + secrets. ### Reliability & operations -- **Backups + restore runbook** — `pg_dump` of `core.*`, storage-root - archive, cron/timer example, and a documented restore path. The restore - must be tested, not just the dump. -- **Retention** — telemetry events, audit log, and webhook deliveries grow - unbounded. Per-project retention settings + a sweeper job. -- **Platform metrics export** — internal counters exist (surfaced in - readiness); expose `/metrics` for Prometheus scraping. -- **Production compose profile** — healthchecks on every service, restart - policies, pinned versions for dependencies (dragonfly is on `latest`). -- **Upgrade runbook** — today it's `git pull && compose up -d`. Document - the safe order (backup → pull → migrate-on-boot → verify) and rollback. +- ~~**Backups + restore runbook**~~ ✅ — `scripts/backup.sh` (pg_dump -Fc + + storage volume tar) and `scripts/restore.sh`; restore verified against a + scratch postgres + volume. Cron/systemd examples in the docs. +- ~~**Retention**~~ ✅ — per-project `retention_{events,audit,webhook}_days` + (migration 00007, 0 = disabled) + hourly `RetentionSweeper`; settings + exposed in the project edit dialog. +- ~~**Platform metrics export**~~ ✅ — `GET /api/v1/metrics` renders + Prometheus text exposition (requests, errors, active, duration sum). +- ~~**Production compose profile**~~ ✅ — healthchecks + `restart: + unless-stopped` on every service, `depends_on: service_healthy`, + Dragonfly pinned to `v2.0.0`. +- ~~**Upgrade runbook**~~ ✅ — DEPLOYMENT_GUIDE.md documents backup → pull → + migrate-on-boot → verify, plus rollback via `goose down`. ### Correctness -- **Pagination audit** — every list endpoint must paginate; verify the - heaviest (audit, events, webhook deliveries) can't return unbounded - rows. -- **DB-to-DB links** — the one open Phase 2 item: DBX transfer between two - saved connections. -- **External object storage** — files live under `BACKEND_STORAGE_ROOT` - on local disk. Acceptable for single-node self-hosting; document the - constraint or add an S3-compatible backend behind the storage service. +- ~~**Pagination audit**~~ ✅ — every `:many` query audited; time-series and + entity lists are naturally bounded; `ListComponents` capped at 500 + (auto-registers from ingest). +- ~~**DB-to-DB links**~~ ✅ — implemented and verified live (postgres → + dragonfly transfer, key templating, TTL). +- ~~**External object storage**~~ ✅ — constraint documented in + DEPLOYMENT_GUIDE.md / PRODUCTION_READINESS.md; S3 backend deferred. ### Testing -- **E2E smoke in CI** — `ci.yml` runs unit tests; add a compose-up job - exercising health, signup/login, project create, and ingest → issue - created. Six Go test files + one frontend file is thin for a platform. -- **Migration tests** — goose up/down against a scratch database in CI. -- **Load sanity** — one `k6`/`vegeta` script against ingest + a hot read - path; publish numbers, don't promise scale. +- ~~**E2E smoke in CI**~~ ✅ — `scripts/smoke-e2e.sh` + `e2e-smoke` job: + health, signup, token, bootstrap/org+project, scoped key create, ingest, + read-only key rejection, metrics. Verified locally against the stack. +- ~~**Migration tests**~~ ✅ — `migrations` job runs goose up → reset → up + against scratch postgres; verified locally. +- ~~**Load sanity**~~ ✅ — `scripts/load-sanity.js` (k6): ingest + events + read, 15 VUs / 30 s. Local run: 6650 requests, ~217 req/s, 0 real + failures, p95 221 ms — most requests beyond the configured 600/min cap + correctly returned 429. ### Distribution -- **arm64 images** — releases build amd64 only; ARM VPS/Pi can't pull. Add - `platforms: linux/amd64,linux/arm64` to the buildx steps. +- ~~**arm64 images**~~ ✅ — release builds `linux/amd64,linux/arm64` via + QEMU + buildx. - Carried forward from Phase 5 follow-ups: macOS/Windows bundle verification, signing + auto-updater. - Optional: SBOM + provenance attestation on release images. ### Docs -- **Self-host guide** — TLS, SMTP, backups, upgrade, and env reference in - one place. `scripts/verify-production-ready.sh` already points at - `DEPLOYMENT_GUIDE.md` / `PRODUCTION_READINESS.md`, which don't exist — - write them or fix the script. +- ~~**Self-host guide**~~ ✅ — `DEPLOYMENT_GUIDE.md` (TLS, SMTP, backups, + upgrades, env reference, storage constraint) and + `PRODUCTION_READINESS.md` written; `verify-production-ready.sh` updated + to check them plus the Phase 6 artifacts. **v1.0 bar:** every Security item done, a restore test actually run, E2E smoke green in CI, and the upgrade runbook followed once on a real diff --git a/apps/auth/src/lib/auth.ts b/apps/auth/src/lib/auth.ts index b0258b6..1c08ca8 100644 --- a/apps/auth/src/lib/auth.ts +++ b/apps/auth/src/lib/auth.ts @@ -75,6 +75,12 @@ export const auth = betterAuth({ }); }, }, + // Enabled unconditionally — better-auth only turns this on when NODE_ENV is + // production, but the shipped .env defaults to development. Credential + // endpoints get 3 req/10s; the IP-level limiter in index.ts sits in front. + rateLimit: { + enabled: true, + }, socialProviders, plugins: [ jwt({ diff --git a/apps/backend/db/migrations/00006_api_key_scopes.sql b/apps/backend/db/migrations/00006_api_key_scopes.sql new file mode 100644 index 0000000..8674d42 --- /dev/null +++ b/apps/backend/db/migrations/00006_api_key_scopes.sql @@ -0,0 +1,6 @@ +-- +goose Up +ALTER TABLE core.api_keys + ADD COLUMN scopes TEXT[] NOT NULL DEFAULT '{admin}'; + +-- +goose Down +ALTER TABLE core.api_keys DROP COLUMN IF EXISTS scopes; diff --git a/apps/backend/db/migrations/00007_retention.sql b/apps/backend/db/migrations/00007_retention.sql new file mode 100644 index 0000000..1fa476e --- /dev/null +++ b/apps/backend/db/migrations/00007_retention.sql @@ -0,0 +1,11 @@ +-- +goose Up +ALTER TABLE core.projects + ADD COLUMN retention_events_days INT NOT NULL DEFAULT 90, + ADD COLUMN retention_audit_days INT NOT NULL DEFAULT 365, + ADD COLUMN retention_webhook_days INT NOT NULL DEFAULT 30; + +-- +goose Down +ALTER TABLE core.projects + DROP COLUMN IF EXISTS retention_events_days, + DROP COLUMN IF EXISTS retention_audit_days, + DROP COLUMN IF EXISTS retention_webhook_days; diff --git a/apps/backend/db/queries/api_keys.sql b/apps/backend/db/queries/api_keys.sql index 7706f40..02c102e 100644 --- a/apps/backend/db/queries/api_keys.sql +++ b/apps/backend/db/queries/api_keys.sql @@ -4,8 +4,9 @@ INSERT INTO core.api_keys ( name, prefix, secret_hash, + scopes, created_by_user_id -) VALUES ($1, $2, $3, $4, $5) +) VALUES ($1, $2, $3, $4, $5, $6) RETURNING *; -- name: ListAPIKeysForProject :many diff --git a/apps/backend/db/queries/projects.sql b/apps/backend/db/queries/projects.sql index 837394b..b774d16 100644 --- a/apps/backend/db/queries/projects.sql +++ b/apps/backend/db/queries/projects.sql @@ -28,7 +28,10 @@ RETURNING *; UPDATE core.projects SET slug = $2, name = $3, - description = $4 + description = $4, + retention_events_days = COALESCE(sqlc.narg('retention_events_days'), retention_events_days), + retention_audit_days = COALESCE(sqlc.narg('retention_audit_days'), retention_audit_days), + retention_webhook_days = COALESCE(sqlc.narg('retention_webhook_days'), retention_webhook_days) WHERE id = $1 RETURNING *; diff --git a/apps/backend/db/queries/retention.sql b/apps/backend/db/queries/retention.sql new file mode 100644 index 0000000..7f4959f --- /dev/null +++ b/apps/backend/db/queries/retention.sql @@ -0,0 +1,33 @@ +-- Per-project retention sweeps. A retention of 0 days disables the sweep for +-- that stream. Pending webhook deliveries are never swept — they may still be +-- retrying. + +-- name: SweepExpiredEvents :execrows +DELETE FROM core.events e +USING core.projects p +WHERE e.project_id = p.id + AND p.retention_events_days > 0 + AND e.ts < NOW() - make_interval(days => p.retention_events_days); + +-- name: SweepExpiredAuditLogs :execrows +DELETE FROM core.audit_logs a +USING core.projects p +WHERE a.project_id = p.id + AND p.retention_audit_days > 0 + AND a.created_at < NOW() - make_interval(days => p.retention_audit_days); + +-- name: SweepExpiredOrgAuditLogs :execrows +-- Org-scoped audit rows (project_id NULL) have no project retention — sweep +-- them on a fixed 365-day floor so they cannot grow unbounded. +DELETE FROM core.audit_logs a +WHERE a.project_id IS NULL + AND a.created_at < NOW() - INTERVAL '365 days'; + +-- name: SweepExpiredWebhookDeliveries :execrows +DELETE FROM core.webhook_deliveries d +USING core.webhooks w +JOIN core.projects p ON p.id = w.project_id +WHERE d.webhook_id = w.id + AND p.retention_webhook_days > 0 + AND d.status <> 'pending' + AND d.created_at < NOW() - make_interval(days => p.retention_webhook_days); diff --git a/apps/backend/db/queries/telemetry.sql b/apps/backend/db/queries/telemetry.sql index 4a41ec4..f980f24 100644 --- a/apps/backend/db/queries/telemetry.sql +++ b/apps/backend/db/queries/telemetry.sql @@ -11,7 +11,8 @@ WHERE project_id = $1 AND name = $2; -- name: ListComponents :many SELECT * FROM core.components WHERE project_id = $1 -ORDER BY name; +ORDER BY name +LIMIT 500; -- name: DeleteComponent :exec DELETE FROM core.components diff --git a/apps/backend/internal/app/app.go b/apps/backend/internal/app/app.go index 8a77deb..1225c46 100644 --- a/apps/backend/internal/app/app.go +++ b/apps/backend/internal/app/app.go @@ -2,12 +2,16 @@ package app import ( "context" + "errors" "fmt" "log/slog" + "net/http" "net/url" "os" + "os/signal" "strconv" "strings" + "syscall" "time" "github.com/gin-gonic/gin" @@ -37,6 +41,8 @@ type App struct { Redis *redis.Client DBX *dbx.Client Platform *services.PlatformService + + sweepCancel context.CancelFunc } func Bootstrap(ctx context.Context) (*App, error) { @@ -157,21 +163,53 @@ func Bootstrap(ctx context.Context) (*App, error) { } handler.Register(router) + sweepCtx, sweepCancel := context.WithCancel(context.Background()) + sweeper := services.NewRetentionSweeper(repo, logger) + go sweeper.Run(sweepCtx) + return &App{ - Config: cfg, - Router: router, - Logger: logger, - DB: dbPool, - Redis: redisClient, - DBX: dbxClient, - Platform: platform, + Config: cfg, + Router: router, + Logger: logger, + DB: dbPool, + Redis: redisClient, + DBX: dbxClient, + Platform: platform, + sweepCancel: sweepCancel, }, nil } func (a *App) Run() error { address := ":" + a.Config.ServerPort - a.Logger.Info("primora backend starting", "address", address) - return a.Router.Run(address) + srv := &http.Server{Addr: address, Handler: a.Router} + + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer stop() + + errCh := make(chan error, 1) + go func() { + a.Logger.Info("primora backend starting", "address", address) + errCh <- srv.ListenAndServe() + }() + + select { + case err := <-errCh: + return err + case <-ctx.Done(): + } + + a.Logger.Info("shutdown signal received, draining in-flight requests") + shutdownCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + if err := srv.Shutdown(shutdownCtx); err != nil { + return fmt.Errorf("graceful shutdown: %w", err) + } + if err := <-errCh; err != nil && !errors.Is(err, http.ErrServerClosed) { + return err + } + // Stop the retention sweeper before main closes the DB pool. + a.sweepCancel() + return nil } // firstPartyDBSeeds turns the platform's own Postgres and Dragonfly URLs into @@ -206,6 +244,9 @@ func firstPartyDBSeeds(cfg config.Config) []services.ManagedDBSeed { } func (a *App) Close() error { + if a.sweepCancel != nil { + a.sweepCancel() + } if a.Platform != nil { a.Platform.Close() } diff --git a/apps/backend/internal/database/db/api_keys.sql.go b/apps/backend/internal/database/db/api_keys.sql.go index 35676ab..d204ffb 100644 --- a/apps/backend/internal/database/db/api_keys.sql.go +++ b/apps/backend/internal/database/db/api_keys.sql.go @@ -18,9 +18,10 @@ INSERT INTO core.api_keys ( name, prefix, secret_hash, + scopes, created_by_user_id -) VALUES ($1, $2, $3, $4, $5) -RETURNING id, project_id, name, prefix, secret_hash, created_by_user_id, last_used_at, revoked_at, created_at +) VALUES ($1, $2, $3, $4, $5, $6) +RETURNING id, project_id, name, prefix, secret_hash, created_by_user_id, last_used_at, revoked_at, created_at, scopes ` type CreateAPIKeyParams struct { @@ -28,6 +29,7 @@ type CreateAPIKeyParams struct { Name string `json:"name"` Prefix string `json:"prefix"` SecretHash []byte `json:"secret_hash"` + Scopes []string `json:"scopes"` CreatedByUserID pgtype.UUID `json:"created_by_user_id"` } @@ -37,6 +39,7 @@ func (q *Queries) CreateAPIKey(ctx context.Context, arg CreateAPIKeyParams) (Cor arg.Name, arg.Prefix, arg.SecretHash, + arg.Scopes, arg.CreatedByUserID, ) var i CoreApiKey @@ -50,12 +53,13 @@ func (q *Queries) CreateAPIKey(ctx context.Context, arg CreateAPIKeyParams) (Cor &i.LastUsedAt, &i.RevokedAt, &i.CreatedAt, + &i.Scopes, ) return i, err } const getAPIKeyByIDForProject = `-- name: GetAPIKeyByIDForProject :one -SELECT id, project_id, name, prefix, secret_hash, created_by_user_id, last_used_at, revoked_at, created_at FROM core.api_keys +SELECT id, project_id, name, prefix, secret_hash, created_by_user_id, last_used_at, revoked_at, created_at, scopes FROM core.api_keys WHERE project_id = $1 AND id = $2 ` @@ -78,13 +82,14 @@ func (q *Queries) GetAPIKeyByIDForProject(ctx context.Context, arg GetAPIKeyByID &i.LastUsedAt, &i.RevokedAt, &i.CreatedAt, + &i.Scopes, ) return i, err } const getAPIKeyByPrefix = `-- name: GetAPIKeyByPrefix :one SELECT - ak.id, ak.project_id, ak.name, ak.prefix, ak.secret_hash, ak.created_by_user_id, ak.last_used_at, ak.revoked_at, ak.created_at, + ak.id, ak.project_id, ak.name, ak.prefix, ak.secret_hash, ak.created_by_user_id, ak.last_used_at, ak.revoked_at, ak.created_at, ak.scopes, p.organization_id FROM core.api_keys ak JOIN core.projects p ON p.id = ak.project_id @@ -101,6 +106,7 @@ type GetAPIKeyByPrefixRow struct { LastUsedAt pgtype.Timestamptz `json:"last_used_at"` RevokedAt pgtype.Timestamptz `json:"revoked_at"` CreatedAt pgtype.Timestamptz `json:"created_at"` + Scopes []string `json:"scopes"` OrganizationID uuid.UUID `json:"organization_id"` } @@ -117,13 +123,14 @@ func (q *Queries) GetAPIKeyByPrefix(ctx context.Context, prefix string) (GetAPIK &i.LastUsedAt, &i.RevokedAt, &i.CreatedAt, + &i.Scopes, &i.OrganizationID, ) return i, err } const listAPIKeysForProject = `-- name: ListAPIKeysForProject :many -SELECT id, project_id, name, prefix, secret_hash, created_by_user_id, last_used_at, revoked_at, created_at FROM core.api_keys +SELECT id, project_id, name, prefix, secret_hash, created_by_user_id, last_used_at, revoked_at, created_at, scopes FROM core.api_keys WHERE project_id = $1 ORDER BY created_at DESC ` @@ -147,6 +154,7 @@ func (q *Queries) ListAPIKeysForProject(ctx context.Context, projectID uuid.UUID &i.LastUsedAt, &i.RevokedAt, &i.CreatedAt, + &i.Scopes, ); err != nil { return nil, err } @@ -164,7 +172,7 @@ SET revoked_at = NOW() WHERE project_id = $1 AND id = $2 AND revoked_at IS NULL -RETURNING id, project_id, name, prefix, secret_hash, created_by_user_id, last_used_at, revoked_at, created_at +RETURNING id, project_id, name, prefix, secret_hash, created_by_user_id, last_used_at, revoked_at, created_at, scopes ` type RevokeAPIKeyParams struct { @@ -185,6 +193,7 @@ func (q *Queries) RevokeAPIKey(ctx context.Context, arg RevokeAPIKeyParams) (Cor &i.LastUsedAt, &i.RevokedAt, &i.CreatedAt, + &i.Scopes, ) return i, err } diff --git a/apps/backend/internal/database/db/bootstrap.sql.go b/apps/backend/internal/database/db/bootstrap.sql.go index 1fe1901..6a2fd0c 100644 --- a/apps/backend/internal/database/db/bootstrap.sql.go +++ b/apps/backend/internal/database/db/bootstrap.sql.go @@ -24,7 +24,7 @@ new_org_member AS ( new_project AS ( INSERT INTO core.projects (organization_id, slug, name, description) SELECT id, $4, $5, $6 FROM new_org - RETURNING id, organization_id, slug, name, description, created_at + RETURNING id, organization_id, slug, name, description, created_at, retention_events_days, retention_audit_days, retention_webhook_days ), new_project_member AS ( INSERT INTO core.project_members (project_id, user_id, role) diff --git a/apps/backend/internal/database/db/models.go b/apps/backend/internal/database/db/models.go index 7bad57a..ea6c0cb 100644 --- a/apps/backend/internal/database/db/models.go +++ b/apps/backend/internal/database/db/models.go @@ -150,6 +150,7 @@ type CoreApiKey struct { LastUsedAt pgtype.Timestamptz `json:"last_used_at"` RevokedAt pgtype.Timestamptz `json:"revoked_at"` CreatedAt pgtype.Timestamptz `json:"created_at"` + Scopes []string `json:"scopes"` } type CoreAuditLog struct { @@ -273,12 +274,15 @@ type CoreOrganizationMember struct { } type CoreProject struct { - ID uuid.UUID `json:"id"` - OrganizationID uuid.UUID `json:"organization_id"` - Slug string `json:"slug"` - Name string `json:"name"` - Description *string `json:"description"` - CreatedAt pgtype.Timestamptz `json:"created_at"` + ID uuid.UUID `json:"id"` + OrganizationID uuid.UUID `json:"organization_id"` + Slug string `json:"slug"` + Name string `json:"name"` + Description *string `json:"description"` + CreatedAt pgtype.Timestamptz `json:"created_at"` + RetentionEventsDays int32 `json:"retention_events_days"` + RetentionAuditDays int32 `json:"retention_audit_days"` + RetentionWebhookDays int32 `json:"retention_webhook_days"` } type CoreProjectInvitation struct { diff --git a/apps/backend/internal/database/db/projects.sql.go b/apps/backend/internal/database/db/projects.sql.go index b943524..7ecc852 100644 --- a/apps/backend/internal/database/db/projects.sql.go +++ b/apps/backend/internal/database/db/projects.sql.go @@ -59,7 +59,7 @@ INSERT INTO core.projects ( name, description ) VALUES ($1, $2, $3, $4) -RETURNING id, organization_id, slug, name, description, created_at +RETURNING id, organization_id, slug, name, description, created_at, retention_events_days, retention_audit_days, retention_webhook_days ` type CreateProjectParams struct { @@ -84,6 +84,9 @@ func (q *Queries) CreateProject(ctx context.Context, arg CreateProjectParams) (C &i.Name, &i.Description, &i.CreatedAt, + &i.RetentionEventsDays, + &i.RetentionAuditDays, + &i.RetentionWebhookDays, ) return i, err } @@ -91,7 +94,7 @@ func (q *Queries) CreateProject(ctx context.Context, arg CreateProjectParams) (C const deleteProjectByID = `-- name: DeleteProjectByID :one DELETE FROM core.projects WHERE id = $1 -RETURNING id, organization_id, slug, name, description, created_at +RETURNING id, organization_id, slug, name, description, created_at, retention_events_days, retention_audit_days, retention_webhook_days ` func (q *Queries) DeleteProjectByID(ctx context.Context, id uuid.UUID) (CoreProject, error) { @@ -104,12 +107,15 @@ func (q *Queries) DeleteProjectByID(ctx context.Context, id uuid.UUID) (CoreProj &i.Name, &i.Description, &i.CreatedAt, + &i.RetentionEventsDays, + &i.RetentionAuditDays, + &i.RetentionWebhookDays, ) return i, err } const getProjectByID = `-- name: GetProjectByID :one -SELECT id, organization_id, slug, name, description, created_at FROM core.projects +SELECT id, organization_id, slug, name, description, created_at, retention_events_days, retention_audit_days, retention_webhook_days FROM core.projects WHERE id = $1 ` @@ -123,6 +129,9 @@ func (q *Queries) GetProjectByID(ctx context.Context, id uuid.UUID) (CoreProject &i.Name, &i.Description, &i.CreatedAt, + &i.RetentionEventsDays, + &i.RetentionAuditDays, + &i.RetentionWebhookDays, ) return i, err } @@ -327,7 +336,7 @@ func (q *Queries) ListProjectMembers(ctx context.Context, projectID uuid.UUID) ( const listProjectsForOrganization = `-- name: ListProjectsForOrganization :many SELECT - p.id, p.organization_id, p.slug, p.name, p.description, p.created_at, + p.id, p.organization_id, p.slug, p.name, p.description, p.created_at, p.retention_events_days, p.retention_audit_days, p.retention_webhook_days, pm.role AS membership_role FROM core.projects p LEFT JOIN core.project_members pm @@ -350,13 +359,16 @@ type ListProjectsForOrganizationParams struct { } type ListProjectsForOrganizationRow struct { - ID uuid.UUID `json:"id"` - OrganizationID uuid.UUID `json:"organization_id"` - Slug string `json:"slug"` - Name string `json:"name"` - Description *string `json:"description"` - CreatedAt pgtype.Timestamptz `json:"created_at"` - MembershipRole NullCoreProjectRole `json:"membership_role"` + ID uuid.UUID `json:"id"` + OrganizationID uuid.UUID `json:"organization_id"` + Slug string `json:"slug"` + Name string `json:"name"` + Description *string `json:"description"` + CreatedAt pgtype.Timestamptz `json:"created_at"` + RetentionEventsDays int32 `json:"retention_events_days"` + RetentionAuditDays int32 `json:"retention_audit_days"` + RetentionWebhookDays int32 `json:"retention_webhook_days"` + MembershipRole NullCoreProjectRole `json:"membership_role"` } func (q *Queries) ListProjectsForOrganization(ctx context.Context, arg ListProjectsForOrganizationParams) ([]ListProjectsForOrganizationRow, error) { @@ -375,6 +387,9 @@ func (q *Queries) ListProjectsForOrganization(ctx context.Context, arg ListProje &i.Name, &i.Description, &i.CreatedAt, + &i.RetentionEventsDays, + &i.RetentionAuditDays, + &i.RetentionWebhookDays, &i.MembershipRole, ); err != nil { return nil, err @@ -416,16 +431,22 @@ const updateProjectByID = `-- name: UpdateProjectByID :one UPDATE core.projects SET slug = $2, name = $3, - description = $4 + description = $4, + retention_events_days = COALESCE($5, retention_events_days), + retention_audit_days = COALESCE($6, retention_audit_days), + retention_webhook_days = COALESCE($7, retention_webhook_days) WHERE id = $1 -RETURNING id, organization_id, slug, name, description, created_at +RETURNING id, organization_id, slug, name, description, created_at, retention_events_days, retention_audit_days, retention_webhook_days ` type UpdateProjectByIDParams struct { - ID uuid.UUID `json:"id"` - Slug string `json:"slug"` - Name string `json:"name"` - Description *string `json:"description"` + ID uuid.UUID `json:"id"` + Slug string `json:"slug"` + Name string `json:"name"` + Description *string `json:"description"` + RetentionEventsDays *int32 `json:"retention_events_days"` + RetentionAuditDays *int32 `json:"retention_audit_days"` + RetentionWebhookDays *int32 `json:"retention_webhook_days"` } func (q *Queries) UpdateProjectByID(ctx context.Context, arg UpdateProjectByIDParams) (CoreProject, error) { @@ -434,6 +455,9 @@ func (q *Queries) UpdateProjectByID(ctx context.Context, arg UpdateProjectByIDPa arg.Slug, arg.Name, arg.Description, + arg.RetentionEventsDays, + arg.RetentionAuditDays, + arg.RetentionWebhookDays, ) var i CoreProject err := row.Scan( @@ -443,6 +467,9 @@ func (q *Queries) UpdateProjectByID(ctx context.Context, arg UpdateProjectByIDPa &i.Name, &i.Description, &i.CreatedAt, + &i.RetentionEventsDays, + &i.RetentionAuditDays, + &i.RetentionWebhookDays, ) return i, err } diff --git a/apps/backend/internal/database/db/querier.go b/apps/backend/internal/database/db/querier.go index b908794..843b5e9 100644 --- a/apps/backend/internal/database/db/querier.go +++ b/apps/backend/internal/database/db/querier.go @@ -100,6 +100,15 @@ type Querier interface { RemoveProjectMember(ctx context.Context, arg RemoveProjectMemberParams) (CoreProjectMember, error) RemoveProjectMembershipsForOrganizationUser(ctx context.Context, arg RemoveProjectMembershipsForOrganizationUserParams) error RevokeAPIKey(ctx context.Context, arg RevokeAPIKeyParams) (CoreApiKey, error) + SweepExpiredAuditLogs(ctx context.Context) (int64, error) + // Per-project retention sweeps. A retention of 0 days disables the sweep for + // that stream. Pending webhook deliveries are never swept — they may still be + // retrying. + SweepExpiredEvents(ctx context.Context) (int64, error) + // Org-scoped audit rows (project_id NULL) have no project retention — sweep + // them on a fixed 365-day floor so they cannot grow unbounded. + SweepExpiredOrgAuditLogs(ctx context.Context) (int64, error) + SweepExpiredWebhookDeliveries(ctx context.Context) (int64, error) TouchAPIKey(ctx context.Context, id uuid.UUID) error UpdateBucketByID(ctx context.Context, arg UpdateBucketByIDParams) (CoreBucket, error) UpdateCollection(ctx context.Context, arg UpdateCollectionParams) (CoreCollection, error) diff --git a/apps/backend/internal/database/db/retention.sql.go b/apps/backend/internal/database/db/retention.sql.go new file mode 100644 index 0000000..47d4096 --- /dev/null +++ b/apps/backend/internal/database/db/retention.sql.go @@ -0,0 +1,80 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.29.0 +// source: retention.sql + +package db + +import ( + "context" +) + +const sweepExpiredAuditLogs = `-- name: SweepExpiredAuditLogs :execrows +DELETE FROM core.audit_logs a +USING core.projects p +WHERE a.project_id = p.id + AND p.retention_audit_days > 0 + AND a.created_at < NOW() - make_interval(days => p.retention_audit_days) +` + +func (q *Queries) SweepExpiredAuditLogs(ctx context.Context) (int64, error) { + result, err := q.db.Exec(ctx, sweepExpiredAuditLogs) + if err != nil { + return 0, err + } + return result.RowsAffected(), nil +} + +const sweepExpiredEvents = `-- name: SweepExpiredEvents :execrows + +DELETE FROM core.events e +USING core.projects p +WHERE e.project_id = p.id + AND p.retention_events_days > 0 + AND e.ts < NOW() - make_interval(days => p.retention_events_days) +` + +// Per-project retention sweeps. A retention of 0 days disables the sweep for +// that stream. Pending webhook deliveries are never swept — they may still be +// retrying. +func (q *Queries) SweepExpiredEvents(ctx context.Context) (int64, error) { + result, err := q.db.Exec(ctx, sweepExpiredEvents) + if err != nil { + return 0, err + } + return result.RowsAffected(), nil +} + +const sweepExpiredOrgAuditLogs = `-- name: SweepExpiredOrgAuditLogs :execrows +DELETE FROM core.audit_logs a +WHERE a.project_id IS NULL + AND a.created_at < NOW() - INTERVAL '365 days' +` + +// Org-scoped audit rows (project_id NULL) have no project retention — sweep +// them on a fixed 365-day floor so they cannot grow unbounded. +func (q *Queries) SweepExpiredOrgAuditLogs(ctx context.Context) (int64, error) { + result, err := q.db.Exec(ctx, sweepExpiredOrgAuditLogs) + if err != nil { + return 0, err + } + return result.RowsAffected(), nil +} + +const sweepExpiredWebhookDeliveries = `-- name: SweepExpiredWebhookDeliveries :execrows +DELETE FROM core.webhook_deliveries d +USING core.webhooks w +JOIN core.projects p ON p.id = w.project_id +WHERE d.webhook_id = w.id + AND p.retention_webhook_days > 0 + AND d.status <> 'pending' + AND d.created_at < NOW() - make_interval(days => p.retention_webhook_days) +` + +func (q *Queries) SweepExpiredWebhookDeliveries(ctx context.Context) (int64, error) { + result, err := q.db.Exec(ctx, sweepExpiredWebhookDeliveries) + if err != nil { + return 0, err + } + return result.RowsAffected(), nil +} diff --git a/apps/backend/internal/database/db/telemetry.sql.go b/apps/backend/internal/database/db/telemetry.sql.go index c9175e3..202f3ee 100644 --- a/apps/backend/internal/database/db/telemetry.sql.go +++ b/apps/backend/internal/database/db/telemetry.sql.go @@ -239,6 +239,7 @@ const listComponents = `-- name: ListComponents :many SELECT id, project_id, name, kind, meta, created_at FROM core.components WHERE project_id = $1 ORDER BY name +LIMIT 500 ` func (q *Queries) ListComponents(ctx context.Context, projectID uuid.UUID) ([]CoreComponent, error) { diff --git a/apps/backend/internal/handlers/databases.go b/apps/backend/internal/handlers/databases.go index 8e1c487..9612306 100644 --- a/apps/backend/internal/handlers/databases.go +++ b/apps/backend/internal/handlers/databases.go @@ -204,6 +204,23 @@ func (h *HTTPHandler) dbxRedis(c *gin.Context) { c.JSON(http.StatusOK, result) } +func (h *HTTPHandler) dbxTransfer(c *gin.Context) { + actor, projectID, connectionID, ok := h.actorProjectConn(c) + if !ok { + return + } + var body services.TransferInput + if !h.bindAndValidate(c, &body) { + return + } + result, err := h.Platform.TransferDBRows(c.Request.Context(), actor, projectID, connectionID, body, middleware.RequestIDFromContext(c)) + if err != nil { + h.dbxError(c, err) + return + } + c.JSON(http.StatusOK, result) +} + // actorAndProject resolves the actor and :projectID param. func (h *HTTPHandler) actorAndProject(c *gin.Context) (*models.Actor, uuid.UUID, bool) { actor, ok := middleware.RequireActor(c) diff --git a/apps/backend/internal/handlers/http.go b/apps/backend/internal/handlers/http.go index f1338af..cb692e4 100644 --- a/apps/backend/internal/handlers/http.go +++ b/apps/backend/internal/handlers/http.go @@ -29,6 +29,7 @@ func (h *HTTPHandler) Register(router *gin.Engine) { router.GET("/api/v1/health/liveness", h.liveness) router.GET("/api/v1/health/readiness", h.readiness) router.GET("/api/v1/openapi.yaml", h.openapi) + router.GET("/api/v1/metrics", h.metrics) api := router.Group("/api/v1") api.GET("/me", h.me) @@ -91,6 +92,7 @@ func (h *HTTPHandler) Register(router *gin.Engine) { api.GET("/projects/:projectID/db-connections/:connectionID/foreign-keys", h.dbxForeignKeys) api.POST("/projects/:projectID/db-connections/:connectionID/query", h.dbxQuery) api.POST("/projects/:projectID/db-connections/:connectionID/redis", h.dbxRedis) + api.POST("/projects/:projectID/db-connections/:connectionID/transfer", h.dbxTransfer) api.GET("/projects/:projectID/integrations", h.listIntegrations) api.POST("/projects/:projectID/integrations", h.createIntegration) api.DELETE("/projects/:projectID/integrations/:integrationID", h.deleteIntegration) @@ -123,6 +125,12 @@ func (h *HTTPHandler) openapi(c *gin.Context) { c.File("openapi/openapi.yaml") } +// metrics is intentionally unauthenticated — Prometheus scrapes it. Restrict +// it at the edge if the deployment is public-facing. +func (h *HTTPHandler) metrics(c *gin.Context) { + c.Data(http.StatusOK, "text/plain; version=0.0.4; charset=utf-8", []byte(h.Metrics.RenderPrometheus())) +} + func (h *HTTPHandler) me(c *gin.Context) { actor, ok := middleware.RequireActor(c) if !ok { diff --git a/apps/backend/internal/handlers/serializers.go b/apps/backend/internal/handlers/serializers.go index 3b8369b..cc7e6d8 100644 --- a/apps/backend/internal/handlers/serializers.go +++ b/apps/backend/internal/handlers/serializers.go @@ -17,12 +17,15 @@ type organizationMembershipResponse struct { } type projectResponse struct { - ID string `json:"id"` - OrganizationID string `json:"organization_id"` - Slug string `json:"slug"` - Name string `json:"name"` - Description *string `json:"description"` - MembershipRole *string `json:"membership_role"` + ID string `json:"id"` + OrganizationID string `json:"organization_id"` + Slug string `json:"slug"` + Name string `json:"name"` + Description *string `json:"description"` + MembershipRole *string `json:"membership_role"` + RetentionEventsDays int32 `json:"retention_events_days"` + RetentionAuditDays int32 `json:"retention_audit_days"` + RetentionWebhookDays int32 `json:"retention_webhook_days"` } type apiKeyResponse struct { @@ -30,6 +33,7 @@ type apiKeyResponse struct { ProjectID string `json:"project_id"` Name string `json:"name"` Prefix string `json:"prefix"` + Scopes []string `json:"scopes"` LastUsedAt *time.Time `json:"last_used_at"` RevokedAt *time.Time `json:"revoked_at"` } @@ -114,23 +118,29 @@ func toProjectFromRow(row db.ListProjectsForOrganizationRow) projectResponse { membershipRole = &role } return projectResponse{ - ID: row.ID.String(), - OrganizationID: row.OrganizationID.String(), - Slug: row.Slug, - Name: row.Name, - Description: row.Description, - MembershipRole: membershipRole, + ID: row.ID.String(), + OrganizationID: row.OrganizationID.String(), + Slug: row.Slug, + Name: row.Name, + Description: row.Description, + MembershipRole: membershipRole, + RetentionEventsDays: row.RetentionEventsDays, + RetentionAuditDays: row.RetentionAuditDays, + RetentionWebhookDays: row.RetentionWebhookDays, } } func toProjectFromCore(row db.CoreProject) projectResponse { return projectResponse{ - ID: row.ID.String(), - OrganizationID: row.OrganizationID.String(), - Slug: row.Slug, - Name: row.Name, - Description: row.Description, - MembershipRole: nil, + ID: row.ID.String(), + OrganizationID: row.OrganizationID.String(), + Slug: row.Slug, + Name: row.Name, + Description: row.Description, + MembershipRole: nil, + RetentionEventsDays: row.RetentionEventsDays, + RetentionAuditDays: row.RetentionAuditDays, + RetentionWebhookDays: row.RetentionWebhookDays, } } @@ -148,6 +158,7 @@ func toAPIKeyResponse(row db.CoreApiKey) apiKeyResponse { ProjectID: row.ProjectID.String(), Name: row.Name, Prefix: row.Prefix, + Scopes: row.Scopes, LastUsedAt: timestamptzPtr(row.LastUsedAt), RevokedAt: timestamptzPtr(row.RevokedAt), } diff --git a/apps/backend/internal/middleware/context.go b/apps/backend/internal/middleware/context.go index ecccd30..c9a6589 100644 --- a/apps/backend/internal/middleware/context.go +++ b/apps/backend/internal/middleware/context.go @@ -125,6 +125,10 @@ func (m AuthMiddleware) ResolveActor() gin.HandlerFunc { if !m.enforceRateLimit(c, "apikey", actor.APIKeyPrefix, m.RateLimits.APIKeyPerMinute, "API key rate limit exceeded") { return } + if need := requiredAPIKeyScope(c.Request.Method, c.Request.URL.Path); !actor.HasScope(need) { + apperrors.Abort(c, http.StatusForbidden, "insufficient_scope", "api key lacks the required scope: "+need) + return + } c.Set(actorKey, actor) } c.Next() @@ -264,9 +268,36 @@ func (m AuthMiddleware) resolveAPIKeyActor(ctx context.Context, rawKey string) ( OrganizationID: &orgID, APIKeyID: &apiKeyID, APIKeyPrefix: prefix, + Scopes: row.Scopes, }, nil } +// requiredAPIKeyScope maps a request to the scope an API key must hold. +// admin-only surfaces: org management, invitations, bootstrap, project +// settings/members, and API key minting (a write key must never mint keys). +func requiredAPIKeyScope(method, path string) string { + switch method { + case http.MethodGet, http.MethodHead, http.MethodOptions: + return "read" + } + if path == "/api/v1/ingest" { + return "ingest" + } + rest := strings.TrimPrefix(path, "/api/v1/") + segs := strings.SplitN(rest, "/", 3) + switch segs[0] { + case "organizations", "invitations", "bootstrap": + return "admin" + case "projects": + if len(segs) < 3 || + strings.HasPrefix(segs[2], "api-keys") || + strings.HasPrefix(segs[2], "members") { + return "admin" + } + } + return "write" +} + func PgUUID(id uuid.UUID) pgtype.UUID { return pgtype.UUID{Bytes: id, Valid: true} } diff --git a/apps/backend/internal/middleware/context_test.go b/apps/backend/internal/middleware/context_test.go new file mode 100644 index 0000000..1f340bf --- /dev/null +++ b/apps/backend/internal/middleware/context_test.go @@ -0,0 +1,66 @@ +package middleware + +import ( + "net/http" + "testing" + + "github.com/tdvorak/primora/apps/backend/internal/models" +) + +func TestRequiredAPIKeyScope(t *testing.T) { + cases := []struct { + method, path, want string + }{ + {http.MethodPost, "/api/v1/ingest", "ingest"}, + {http.MethodOptions, "/api/v1/ingest", "read"}, // preflight passes through CORS + {http.MethodGet, "/api/v1/projects/x/events", "read"}, + {http.MethodGet, "/api/v1/projects/x/api-keys", "read"}, + {http.MethodPost, "/api/v1/projects/x/api-keys", "admin"}, + {http.MethodDelete, "/api/v1/projects/x/api-keys/y", "admin"}, + {http.MethodPatch, "/api/v1/projects/x/members/y", "admin"}, + {http.MethodPatch, "/api/v1/projects/x", "admin"}, + {http.MethodDelete, "/api/v1/projects/x", "admin"}, + {http.MethodPatch, "/api/v1/organizations/x", "admin"}, + {http.MethodDelete, "/api/v1/organizations/x", "admin"}, + {http.MethodPost, "/api/v1/organizations/x/invitations", "admin"}, + {http.MethodPost, "/api/v1/bootstrap", "admin"}, + {http.MethodPost, "/api/v1/projects/x/buckets", "write"}, + {http.MethodPost, "/api/v1/projects/x/db-connections/y/transfer", "write"}, + {http.MethodDelete, "/api/v1/buckets/x", "write"}, + {http.MethodPost, "/api/v1/projects/x/deploy-markers", "write"}, + } + for _, tc := range cases { + if got := requiredAPIKeyScope(tc.method, tc.path); got != tc.want { + t.Errorf("%s %s: got %q, want %q", tc.method, tc.path, got, tc.want) + } + } +} + +func TestActorHasScope(t *testing.T) { + key := func(scopes ...string) *models.Actor { + return &models.Actor{Type: models.ActorTypeAPIKey, Scopes: scopes} + } + user := &models.Actor{Type: models.ActorTypeUser} + + if !user.HasScope("admin") { + t.Error("user actors bypass scope checks") + } + if !key("admin").HasScope("read") { + t.Error("admin should satisfy every requirement") + } + if !key("write").HasScope("read") || !key("write").HasScope("ingest") { + t.Error("write should imply read and ingest") + } + if key("write").HasScope("admin") { + t.Error("write must not imply admin") + } + if key("read").HasScope("write") || key("read").HasScope("ingest") { + t.Error("read must not imply write or ingest") + } + if !key("ingest").HasScope("ingest") || key("ingest").HasScope("read") { + t.Error("ingest keys only reach /ingest") + } + if key().HasScope("read") { + t.Error("scopeless key has no access") + } +} diff --git a/apps/backend/internal/models/actor.go b/apps/backend/internal/models/actor.go index 7f7390a..1077ceb 100644 --- a/apps/backend/internal/models/actor.go +++ b/apps/backend/internal/models/actor.go @@ -21,6 +21,7 @@ type Actor struct { OrganizationID *uuid.UUID APIKeyID *uuid.UUID APIKeyPrefix string + Scopes []string } func (a *Actor) IsUser() bool { @@ -30,3 +31,35 @@ func (a *Actor) IsUser() bool { func (a *Actor) IsAPIKey() bool { return a != nil && a.Type == ActorTypeAPIKey } + +// HasScope reports whether the actor may perform work requiring `need`. +// User actors are always allowed — scopes constrain API keys only. +// Scope semantics: admin unlocks everything; write unlocks read + ingest + +// non-admin mutations; read and ingest unlock only themselves. +func (a *Actor) HasScope(need string) bool { + if a == nil { + return false + } + if a.IsUser() { + return true + } + for _, s := range a.Scopes { + switch s { + case "admin": + return true + case "write": + if need == "read" || need == "write" || need == "ingest" { + return true + } + case "read": + if need == "read" { + return true + } + case "ingest": + if need == "ingest" { + return true + } + } + } + return false +} diff --git a/apps/backend/internal/observability/metrics.go b/apps/backend/internal/observability/metrics.go index d609ec2..630bf9a 100644 --- a/apps/backend/internal/observability/metrics.go +++ b/apps/backend/internal/observability/metrics.go @@ -1,6 +1,7 @@ package observability import ( + "fmt" "sync" "time" ) @@ -43,12 +44,12 @@ func (m *Metrics) DecrementActive() { func (m *Metrics) GetStats() map[string]any { m.mu.RLock() defer m.mu.RUnlock() - + avgResponseTime := int64(0) if m.requestCount > 0 { avgResponseTime = m.totalResponseTime.Milliseconds() / m.requestCount } - + return map[string]any{ "total_requests": m.requestCount, "total_errors": m.errorCount, @@ -56,3 +57,24 @@ func (m *Metrics) GetStats() map[string]any { "avg_response_time_ms": avgResponseTime, } } + +// RenderPrometheus emits the collector in Prometheus text exposition format. +// Duration is exposed as sum+count so scrapers can compute real averages. +func (m *Metrics) RenderPrometheus() string { + m.mu.RLock() + defer m.mu.RUnlock() + return fmt.Sprintf(`# HELP primora_http_requests_total Total HTTP requests handled. +# TYPE primora_http_requests_total counter +primora_http_requests_total %d +# HELP primora_http_request_errors_total HTTP requests that ended in a 5xx response. +# TYPE primora_http_request_errors_total counter +primora_http_request_errors_total %d +# HELP primora_http_active_requests Requests currently in flight. +# TYPE primora_http_active_requests gauge +primora_http_active_requests %d +# HELP primora_http_request_duration_ms_sum Sum of request durations in milliseconds. +# TYPE primora_http_request_duration_ms_sum counter +primora_http_request_duration_ms_sum %d +`, + m.requestCount, m.errorCount, m.activeRequests, m.totalResponseTime.Milliseconds()) +} diff --git a/apps/backend/internal/services/databases.go b/apps/backend/internal/services/databases.go index 694c033..c13da28 100644 --- a/apps/backend/internal/services/databases.go +++ b/apps/backend/internal/services/databases.go @@ -500,3 +500,223 @@ func (s *PlatformService) ExecuteDBXRedis(ctx context.Context, actor *models.Act } return map[string]any{"output": text}, nil } + +// TransferInput describes a DB-to-DB link: rows read from the source +// connection are written to the target connection. +type TransferInput struct { + TargetConnectionID string `json:"target_connection_id" validate:"required,uuid"` + Query string `json:"query" validate:"required"` + Database string `json:"database"` + // SQL targets: destination table (schema-qualified allowed). + TargetTable string `json:"target_table"` + // Redis targets: SET key template with {column} placeholders; the value is + // value_column's cell, or the whole row as JSON when omitted. + KeyPattern string `json:"key_pattern"` + ValueColumn string `json:"value_column"` + // Optional TTL in seconds applied to each Redis SET. + TTLSeconds *int `json:"ttl_seconds" validate:"omitempty,min=0"` + Limit int `json:"limit" validate:"omitempty,min=1,max=1000"` +} + +const transferMaxRows = 1000 + +// TransferDBRows moves data between two saved connections of the same +// project. Both connections must accept write-level access. +func (s *PlatformService) TransferDBRows(ctx context.Context, actor *models.Actor, projectID, sourceID uuid.UUID, input TransferInput, requestID string) (map[string]any, error) { + source, err := s.connForTool(ctx, actor, projectID, sourceID, true) + if err != nil { + return nil, err + } + targetID, err := uuid.Parse(input.TargetConnectionID) + if err != nil { + return nil, errors.New("target_connection_id must be a uuid") + } + target, err := s.connForTool(ctx, actor, projectID, targetID, true) + if err != nil { + return nil, err + } + + limit := input.Limit + if limit <= 0 || limit > transferMaxRows { + limit = transferMaxRows + } + args := map[string]any{"connection_name": dbxConnectionName(projectID, source.Name), "sql": input.Query} + if input.Database != "" { + args["database"] = input.Database + } + text, err := s.dbx.Call(ctx, "dbx_execute_query", args) + if err != nil { + return nil, err + } + table, _ := dbx.ParseMDTable(text) + if table == nil || len(table.Columns) == 0 { + return nil, errors.New("source query did not return a result table") + } + rows := table.Rows + truncated := false + if len(rows) > limit { + rows = rows[:limit] + truncated = true + } + + var transferred int + if target.DbType == "redis" { + transferred, err = s.transferToRedis(ctx, projectID, target, table.Columns, rows, input) + } else { + transferred, err = s.transferToSQL(ctx, projectID, target, table.Columns, rows, input) + } + if err != nil { + return nil, err + } + + project, perr := s.repo.Queries().GetProjectByID(ctx, projectID) + if perr == nil { + _, _ = s.repo.Queries().CreateAuditLog(ctx, newAuditParams(project.OrganizationID, projectID, actor, requestID, "db_connection.transferred", "db_connection", source.ID.String(), map[string]any{ + "source": source.Name, + "target": target.Name, + "transferred": transferred, + "truncated": truncated, + })) + } + return map[string]any{ + "transferred": transferred, + "truncated": truncated, + "source": source.Name, + "target": target.Name, + }, nil +} + +// transferToSQL INSERTs rows into target_table in batches of 100. Cell values +// arrive as text from DBX's markdown output; literal "NULL" maps to SQL NULL. +func (s *PlatformService) transferToSQL(ctx context.Context, projectID uuid.UUID, target db.CoreDbConnection, cols []string, rows [][]string, input TransferInput) (int, error) { + tableIdent, ok := safeDottedIdent(input.TargetTable) + if !ok { + return 0, errors.New("target_table must be a plain identifier or schema.table") + } + colIdents := make([]string, len(cols)) + for i, col := range cols { + ident, ok := safeIdent(col) + if !ok { + return 0, fmt.Errorf("source column %q is not a usable identifier", col) + } + colIdents[i] = `"` + ident + `"` + } + + transferred := 0 + const batch = 100 + for start := 0; start < len(rows); start += batch { + end := min(start+batch, len(rows)) + var values strings.Builder + values.WriteString("INSERT INTO " + tableIdent + " (" + strings.Join(colIdents, ", ") + ") VALUES ") + for i, row := range rows[start:end] { + if i > 0 { + values.WriteString(", ") + } + values.WriteString("(") + for j := range cols { + if j > 0 { + values.WriteString(", ") + } + cell := "" + if j < len(row) { + cell = row[j] + } + if cell == "NULL" { + values.WriteString("NULL") + } else { + values.WriteString(sqlQuote(cell)) + } + } + values.WriteString(")") + } + if _, err := s.dbx.Call(ctx, "dbx_execute_query", map[string]any{ + "connection_name": dbxConnectionName(projectID, target.Name), + "sql": values.String(), + }); err != nil { + return transferred, fmt.Errorf("insert batch at row %d: %w", start, err) + } + transferred += end - start + } + return transferred, nil +} + +// transferToRedis SETs one key per row. Placeholders like {email} in +// key_pattern are replaced with that row's cell value. +var keyPlaceholder = regexp.MustCompile(`\{([A-Za-z0-9_]+)\}`) + +func (s *PlatformService) transferToRedis(ctx context.Context, projectID uuid.UUID, target db.CoreDbConnection, cols []string, rows [][]string, input TransferInput) (int, error) { + if input.KeyPattern == "" { + return 0, errors.New("key_pattern is required for redis targets") + } + colIndex := map[string]int{} + for i, col := range cols { + colIndex[col] = i + } + for _, ph := range keyPlaceholder.FindAllStringSubmatch(input.KeyPattern, -1) { + if _, ok := colIndex[ph[1]]; !ok { + return 0, fmt.Errorf("key_pattern references unknown column %q", ph[1]) + } + } + if input.ValueColumn != "" { + if _, ok := colIndex[input.ValueColumn]; !ok { + return 0, fmt.Errorf("value_column %q not in source columns", input.ValueColumn) + } + } + + transferred := 0 + for _, row := range rows { + key := keyPlaceholder.ReplaceAllStringFunc(input.KeyPattern, func(m string) string { + col := keyPlaceholder.FindStringSubmatch(m)[1] + if i, ok := colIndex[col]; ok && i < len(row) { + return row[i] + } + return "" + }) + var value string + if input.ValueColumn != "" { + value = row[colIndex[input.ValueColumn]] + } else { + obj := map[string]string{} + for i, col := range cols { + if i < len(row) { + obj[col] = row[i] + } + } + raw, _ := json.Marshal(obj) + value = string(raw) + } + cmd := "SET " + redisQuote(key) + " " + redisQuote(value) + if input.TTLSeconds != nil && *input.TTLSeconds > 0 { + cmd += fmt.Sprintf(" EX %d", *input.TTLSeconds) + } + if _, err := s.dbx.Call(ctx, "dbx_execute_redis_command", map[string]any{ + "connection_name": dbxConnectionName(projectID, target.Name), + "command": cmd, + }); err != nil { + return transferred, fmt.Errorf("SET %q: %w", key, err) + } + transferred++ + } + return transferred, nil +} + +var dottedIdent = regexp.MustCompile(`^[A-Za-z0-9_]+(\.[A-Za-z0-9_]+)?$`) + +func safeDottedIdent(s string) (string, bool) { + if !dottedIdent.MatchString(s) { + return "", false + } + parts := strings.Split(s, ".") + for i := range parts { + parts[i] = `"` + parts[i] + `"` + } + return strings.Join(parts, "."), true +} + +func sqlQuote(v string) string { + return "'" + strings.ReplaceAll(v, "'", "''") + "'" +} + +func redisQuote(v string) string { + return `"` + strings.ReplaceAll(strings.ReplaceAll(v, `\`, `\\`), `"`, `\"`) + `"` +} diff --git a/apps/backend/internal/services/platform_service.go b/apps/backend/internal/services/platform_service.go index 8a1c5ca..007f17e 100644 --- a/apps/backend/internal/services/platform_service.go +++ b/apps/backend/internal/services/platform_service.go @@ -67,6 +67,11 @@ type UpdateProjectInput struct { Name string `json:"name" validate:"required,min=2"` Slug string `json:"slug" validate:"required,min=2"` Description *string `json:"description"` + // Retention windows in days; nil leaves the current value, 0 disables + // the sweeper for that stream. + RetentionEventsDays *int32 `json:"retention_events_days" validate:"omitempty,min=0,max=3650"` + RetentionAuditDays *int32 `json:"retention_audit_days" validate:"omitempty,min=0,max=3650"` + RetentionWebhookDays *int32 `json:"retention_webhook_days" validate:"omitempty,min=0,max=3650"` } type CreateInvitationInput struct { @@ -78,7 +83,8 @@ type CreateInvitationInput struct { } type CreateAPIKeyInput struct { - Name string `json:"name" validate:"required,min=2"` + Name string `json:"name" validate:"required,min=2"` + Scopes []string `json:"scopes" validate:"omitempty,dive,oneof=ingest read write admin"` } type CreateBucketInput struct { @@ -736,10 +742,13 @@ func (s *PlatformService) UpdateProject(ctx context.Context, actor *models.Actor return db.CoreProject{}, err } updated, err := s.repo.Queries().UpdateProjectByID(ctx, db.UpdateProjectByIDParams{ - ID: projectID, - Slug: normalizeSlug(input.Slug), - Name: strings.TrimSpace(input.Name), - Description: input.Description, + ID: projectID, + Slug: normalizeSlug(input.Slug), + Name: strings.TrimSpace(input.Name), + Description: input.Description, + RetentionEventsDays: input.RetentionEventsDays, + RetentionAuditDays: input.RetentionAuditDays, + RetentionWebhookDays: input.RetentionWebhookDays, }) if err != nil { return db.CoreProject{}, err @@ -1038,6 +1047,7 @@ func (s *PlatformService) CreateAPIKey(ctx context.Context, actor *models.Actor, if err != nil { return nil, fmt.Errorf("generate api key secret: %w", err) } + scopes := normalizeAPIKeyScopes(input.Scopes) prefix := "prm_" + prefixSuffix rawKey := prefix + "_" + keySuffix secretHash := sha256.Sum256([]byte(rawKey)) @@ -1046,6 +1056,7 @@ func (s *PlatformService) CreateAPIKey(ctx context.Context, actor *models.Actor, Name: strings.TrimSpace(input.Name), Prefix: prefix, SecretHash: secretHash[:], + Scopes: scopes, CreatedByUserID: pgtype.UUID{Bytes: *actor.UserID, Valid: true}, }) if err != nil { @@ -1062,9 +1073,27 @@ func (s *PlatformService) CreateAPIKey(ctx context.Context, actor *models.Actor, "prefix": row.Prefix, "secret": rawKey, "name": row.Name, + "scopes": row.Scopes, }, nil } +// normalizeAPIKeyScopes dedupes the requested scopes. An omitted or empty +// list means a full-access key — the pre-scopes behaviour. +func normalizeAPIKeyScopes(input []string) []string { + if len(input) == 0 { + return []string{"admin"} + } + seen := map[string]bool{} + out := make([]string, 0, len(input)) + for _, s := range input { + if !seen[s] { + seen[s] = true + out = append(out, s) + } + } + return out +} + func (s *PlatformService) RevokeAPIKey(ctx context.Context, actor *models.Actor, projectID, apiKeyID uuid.UUID, requestID string) error { if err := s.requireProjectRole(ctx, actor, projectID, "admin"); err != nil { return err diff --git a/apps/backend/internal/services/retention.go b/apps/backend/internal/services/retention.go new file mode 100644 index 0000000..b574ada --- /dev/null +++ b/apps/backend/internal/services/retention.go @@ -0,0 +1,63 @@ +package services + +import ( + "context" + "log/slog" + "time" + + "github.com/tdvorak/primora/apps/backend/internal/repositories" +) + +// RetentionSweeper deletes telemetry events, audit rows, and webhook +// deliveries that are past each project's retention window. It runs once +// shortly after boot and then hourly — a missed sweep is not a problem, the +// next tick catches it. +type RetentionSweeper struct { + repo *repositories.CoreRepository + logger *slog.Logger + interval time.Duration +} + +func NewRetentionSweeper(repo *repositories.CoreRepository, logger *slog.Logger) *RetentionSweeper { + return &RetentionSweeper{repo: repo, logger: logger, interval: time.Hour} +} + +func (s *RetentionSweeper) Run(ctx context.Context) { + // jarvis: initial delay — let migrations and first-party seeds settle before sweeping. + timer := time.NewTimer(time.Minute) + defer timer.Stop() + for { + select { + case <-ctx.Done(): + return + case <-timer.C: + s.Sweep(ctx) + timer.Reset(s.interval) + } + } +} + +func (s *RetentionSweeper) Sweep(ctx context.Context) { + q := s.repo.Queries() + for _, sweep := range []struct { + name string + fn func(context.Context) (int64, error) + }{ + {"events", q.SweepExpiredEvents}, + {"audit_logs", q.SweepExpiredAuditLogs}, + {"audit_logs_org", q.SweepExpiredOrgAuditLogs}, + {"webhook_deliveries", q.SweepExpiredWebhookDeliveries}, + } { + rows, err := sweep.fn(ctx) + if err != nil { + if ctx.Err() != nil { + return + } + s.logger.Warn("retention sweep failed", "stream", sweep.name, "error", err) + continue + } + if rows > 0 { + s.logger.Info("retention sweep removed rows", "stream", sweep.name, "rows", rows) + } + } +} diff --git a/apps/backend/openapi/openapi.yaml b/apps/backend/openapi/openapi.yaml index ff8f901..2e9724b 100644 --- a/apps/backend/openapi/openapi.yaml +++ b/apps/backend/openapi/openapi.yaml @@ -37,6 +37,18 @@ paths: application/json: schema: $ref: "#/components/schemas/ReadinessStatus" + /metrics: + get: + tags: [Health] + operationId: getMetrics + summary: Prometheus text exposition of platform HTTP metrics + responses: + "200": + description: Prometheus metrics + content: + text/plain: + schema: + type: string /me: get: tags: [Platform] @@ -1092,6 +1104,34 @@ paths: $ref: "#/components/responses/ErrorResponse" "502": $ref: "#/components/responses/ErrorResponse" + /projects/{projectID}/db-connections/{connectionID}/transfer: + post: + tags: [Databases] + operationId: transferDbRows + summary: Move query results from this connection to another saved connection + security: + - bearerAuth: [] + - apiKeyAuth: [] + parameters: + - $ref: "#/components/parameters/ProjectID" + - $ref: "#/components/parameters/ConnectionID" + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/DBTransferRequest" + responses: + "200": + description: Transfer result + content: + application/json: + schema: + $ref: "#/components/schemas/DBTransferResponse" + "404": + $ref: "#/components/responses/ErrorResponse" + "502": + $ref: "#/components/responses/ErrorResponse" /ingest: post: tags: [Telemetry] @@ -1920,6 +1960,21 @@ components: description: type: string nullable: true + retention_events_days: + type: integer + minimum: 0 + maximum: 3650 + description: Telemetry event retention; 0 disables the sweep. + retention_audit_days: + type: integer + minimum: 0 + maximum: 3650 + description: Audit log retention; 0 disables the sweep. + retention_webhook_days: + type: integer + minimum: 0 + maximum: 3650 + description: Webhook delivery retention; 0 disables the sweep. required: [name, slug] Project: type: object @@ -1940,6 +1995,12 @@ components: membership_role: type: string nullable: true + retention_events_days: + type: integer + retention_audit_days: + type: integer + retention_webhook_days: + type: integer required: [id, organization_id, slug, name] ProjectOverview: type: object @@ -2124,6 +2185,15 @@ components: properties: name: type: string + scopes: + type: array + items: + type: string + enum: [ingest, read, write, admin] + description: >- + Access scopes for the key. admin unlocks everything; write unlocks + read + ingest + non-admin mutations; read and ingest unlock only + themselves. Omitted means ["admin"] (full access, legacy behaviour). required: [name] ApiKey: type: object @@ -2138,6 +2208,11 @@ components: type: string prefix: type: string + scopes: + type: array + items: + type: string + enum: [ingest, read, write, admin] last_used_at: type: string format: date-time @@ -2159,6 +2234,11 @@ components: type: string name: type: string + scopes: + type: array + items: + type: string + enum: [ingest, read, write, admin] required: [id, prefix, secret, name] CreateBucketRequest: type: object @@ -2591,6 +2671,52 @@ components: output: type: string required: [output] + DBTransferRequest: + type: object + properties: + target_connection_id: + type: string + format: uuid + query: + type: string + description: SELECT run against the source connection. + database: + type: string + target_table: + type: string + description: Destination table for SQL targets (schema-qualified allowed). + key_pattern: + type: string + description: >- + Redis targets: SET key template with {column} placeholders, e.g. + "user:{id}". + value_column: + type: string + description: >- + Redis targets: column whose cell becomes the value. Omitted stores + the whole row as JSON. + ttl_seconds: + type: integer + minimum: 0 + description: Optional EX TTL applied to each Redis SET. + limit: + type: integer + minimum: 1 + maximum: 1000 + default: 1000 + required: [target_connection_id, query] + DBTransferResponse: + type: object + properties: + transferred: + type: integer + truncated: + type: boolean + source: + type: string + target: + type: string + required: [transferred, truncated, source, target] IngestEvent: type: object properties: diff --git a/apps/frontend/src/App.tsx b/apps/frontend/src/App.tsx index 768156c..1b3ab3a 100644 --- a/apps/frontend/src/App.tsx +++ b/apps/frontend/src/App.tsx @@ -14,6 +14,7 @@ import { UpdateOrganizationRequest, UpdateProjectMemberRoleRequest, UpdateProjectRequest, + CreateApiKeyRequest, CopyBucketObjectRequest, type Bucket, type BucketObject, @@ -180,7 +181,14 @@ export default function App() { }); const [projectInput, setProjectInput] = createSignal({ name: "", slug: "", description: "" }); - const [projectEditInput, setProjectEditInput] = createSignal({ name: "", slug: "", description: "" }); + const [projectEditInput, setProjectEditInput] = createSignal({ + name: "", + slug: "", + description: "", + retentionEventsDays: "", + retentionAuditDays: "", + retentionWebhookDays: "", + }); const [organizationInput, setOrganizationInput] = createSignal({ name: "", slug: "" }); const [organizationEditInput, setOrganizationEditInput] = createSignal({ name: "", slug: "" }); const [invitationInput, setInvitationInput] = createSignal({ @@ -190,6 +198,7 @@ export default function App() { projectRole: CreateInvitationRequest.projectRole.DEVELOPER as string, }); const [apiKeyName, setApiKeyName] = createSignal("Frontend key"); + const [apiKeyScopes, setApiKeyScopes] = createSignal(["admin"]); const [bucketInput, setBucketInput] = createSignal({ name: "", slug: "", @@ -258,7 +267,13 @@ export default function App() { }, ); - const availableProjects = createMemo(() => { + type ProjectListItem = ProjectSummary & { + retentionEventsDays?: number; + retentionAuditDays?: number; + retentionWebhookDays?: number; + }; + + const availableProjects = createMemo(() => { const items = organizationProjects(); if (!items) return activeOrganization()?.projects ?? []; return items.map((project) => ({ @@ -267,10 +282,13 @@ export default function App() { slug: project.slug, description: project.description ?? undefined, membershipRole: project.membership_role ?? undefined, + retentionEventsDays: project.retention_events_days, + retentionAuditDays: project.retention_audit_days, + retentionWebhookDays: project.retention_webhook_days, })); }); - const activeProject = createMemo(() => + const activeProject = createMemo(() => availableProjects().find((item) => item.id === selectedProjectID()), ); const canUpdateOrganization = createMemo(() => { @@ -547,8 +565,15 @@ export default function App() { const project = activeProject(); setProjectEditInput( project - ? { name: project.name, slug: project.slug, description: project.description ?? "" } - : { name: "", slug: "", description: "" }, + ? { + name: project.name, + slug: project.slug, + description: project.description ?? "", + retentionEventsDays: project.retentionEventsDays?.toString() ?? "", + retentionAuditDays: project.retentionAuditDays?.toString() ?? "", + retentionWebhookDays: project.retentionWebhookDays?.toString() ?? "", + } + : { name: "", slug: "", description: "", retentionEventsDays: "", retentionAuditDays: "", retentionWebhookDays: "" }, ); }); @@ -713,6 +738,12 @@ export default function App() { setProjectMessage(""); try { const trimmed = projectEditInput().description.trim(); + const parseRetention = (value: string): number | undefined => { + const raw = value.trim(); + if (raw === "") return undefined; + const n = Number(raw); + return Number.isFinite(n) ? Math.min(3650, Math.max(0, Math.trunc(n))) : undefined; + }; if (isDemo) { await demoService.updateProject({ requestBody: { ...projectEditInput(), description: trimmed || null } }); } else { @@ -722,6 +753,9 @@ export default function App() { name: projectEditInput().name, slug: projectEditInput().slug, description: trimmed.length > 0 ? trimmed : null, + retention_events_days: parseRetention(projectEditInput().retentionEventsDays), + retention_audit_days: parseRetention(projectEditInput().retentionAuditDays), + retention_webhook_days: parseRetention(projectEditInput().retentionWebhookDays), } as UpdateProjectRequest, }); } @@ -1001,7 +1035,14 @@ export default function App() { try { const result = isDemo ? await demoService.createApiKey({ requestBody: { name: apiKeyName() } }) - : await ProjectsService.createApiKey({ projectId: projectID, requestBody: { name: apiKeyName() } }); + : await ProjectsService.createApiKey({ + projectId: projectID, + requestBody: { + name: apiKeyName(), + scopes: apiKeyScopes() as CreateApiKeyRequest["scopes"], + }, + }); + setApiKeyScopes(["admin"]); await refetchAPIKeys(); await refetchAuditLogs(); await refreshProjectOverviewSnapshot(); @@ -1766,6 +1807,7 @@ export default function App() { + setApiKeyScopes((c) => (c.includes(scope) ? c.filter((s) => s !== scope) : [...c, scope])) + } onCreateApiKey={createApiKey} onDeleteApiKey={revokeApiKey} onOrganizationInputChange={(field, value) => setOrganizationInput((c) => ({ ...c, [field]: value }))} diff --git a/apps/frontend/src/lib/demo-mode.ts b/apps/frontend/src/lib/demo-mode.ts index 456e646..8b4a704 100644 --- a/apps/frontend/src/lib/demo-mode.ts +++ b/apps/frontend/src/lib/demo-mode.ts @@ -531,6 +531,9 @@ class DemoService { name: p.name, description: p.description ?? null, membership_role: p.membershipRole ?? null, + retention_events_days: 90, + retention_audit_days: 90, + retention_webhook_days: 90, })); } async listAuthUsers() { await this.delay(); return authUsers; } diff --git a/apps/frontend/src/pages/ProjectsPage.tsx b/apps/frontend/src/pages/ProjectsPage.tsx index 9ec6319..03bf951 100644 --- a/apps/frontend/src/pages/ProjectsPage.tsx +++ b/apps/frontend/src/pages/ProjectsPage.tsx @@ -11,16 +11,22 @@ interface ProjectInput { description: string; } +interface ProjectEditInput extends ProjectInput { + retentionEventsDays: string; + retentionAuditDays: string; + retentionWebhookDays: string; +} + interface ProjectsPageProps { projects: ProjectSummary[]; selectedProjectID?: string; projectInput: ProjectInput; - projectEditInput: ProjectInput; + projectEditInput: ProjectEditInput; projectMessage: string; projectPending: boolean; canUpdateProject: boolean; onProjectInputChange: (field: keyof ProjectInput, value: string) => void; - onProjectEditInputChange: (field: keyof ProjectInput, value: string) => void; + onProjectEditInputChange: (field: keyof ProjectEditInput, value: string) => void; onCreateProject: (event: SubmitEvent) => void; onUpdateProject: (event: SubmitEvent) => void; onDeleteProject: () => void; @@ -252,6 +258,38 @@ export function ProjectsPage(props: ProjectsPageProps) { value={props.projectEditInput.description} onInput={(e) => props.onProjectEditInputChange("description", e.currentTarget.value)} /> +
+ Data retention (days) +

+ Rows older than this are swept hourly. 0 disables a category. Max 3650. +

+
+ props.onProjectEditInputChange("retentionEventsDays", e.currentTarget.value)} + /> + props.onProjectEditInputChange("retentionAuditDays", e.currentTarget.value)} + /> + props.onProjectEditInputChange("retentionWebhookDays", e.currentTarget.value)} + /> +
+
+
+ Scopes +
+ + {(scope) => ( + + )} + +
+
@@ -141,6 +172,7 @@ export function SettingsPage(props: SettingsPageProps) { Name Prefix + Scopes Last used Status @@ -154,6 +186,13 @@ export function SettingsPage(props: SettingsPageProps) { {key.prefix}… + +
+ 0 ? key.scopes : ["admin"]}> + {(scope) => {scope}} + +
+ {key.last_used_at ? props.formatDate(key.last_used_at) : "Never"} diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml new file mode 100644 index 0000000..b4e8244 --- /dev/null +++ b/docker-compose.dev.yml @@ -0,0 +1,28 @@ +# Development overlay — services that must never ship to production. +# Usage: docker compose -f docker-compose.yml -f docker-compose.dev.yml up -d +services: + mailpit: + image: axllent/mailpit:v1.27.9 + ports: + - "${MAILPIT_HTTP_PORT}:8025" + healthcheck: + test: ["CMD-SHELL", "wget -qO- http://localhost:8025/ >/dev/null 2>&1 || exit 1"] + interval: 15s + timeout: 5s + retries: 10 + restart: unless-stopped + + auth: + environment: + SMTP_HOST: mailpit + SMTP_PORT: 1025 + depends_on: + mailpit: + condition: service_started + + nginx: + volumes: + - ./infra/nginx/mailpit.conf:/etc/nginx/primora-extra/mailpit.conf:ro + depends_on: + mailpit: + condition: service_started diff --git a/docker-compose.yml b/docker-compose.yml index 94aac10..19f52d1 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -14,18 +14,18 @@ services: interval: 5s timeout: 5s retries: 20 + restart: unless-stopped dragonfly: - image: docker.dragonflydb.io/dragonflydb/dragonfly:latest + image: docker.dragonflydb.io/dragonflydb/dragonfly:v2.0.0 command: ["--logtostderr", "--proactor_threads=4"] ports: - "${DRAGONFLY_PORT:-6379}:6379" - restart: unless-stopped - - mailpit: - image: axllent/mailpit:latest - ports: - - "${MAILPIT_HTTP_PORT}:8025" + healthcheck: + test: ["CMD-SHELL", "redis-cli -p 6379 ping | grep -q PONG"] + interval: 10s + timeout: 3s + retries: 10 restart: unless-stopped backend: @@ -47,6 +47,12 @@ services: condition: service_started volumes: - backend_storage:/data/storage + healthcheck: + test: ["CMD-SHELL", "curl -fsS http://localhost:8080/api/v1/health/liveness || exit 1"] + interval: 15s + timeout: 5s + retries: 10 + start_period: 20s restart: unless-stopped auth: @@ -61,15 +67,17 @@ services: BETTER_AUTH_URL: http://localhost/auth AUTH_BASE_URL: http://localhost/auth VITE_APP_URL: http://localhost - SMTP_HOST: mailpit - SMTP_PORT: 1025 depends_on: postgres: condition: service_healthy dragonfly: condition: service_started - mailpit: - condition: service_started + healthcheck: + test: ["CMD-SHELL", "wget -qO- http://localhost:3001/health >/dev/null 2>&1 || exit 1"] + interval: 15s + timeout: 5s + retries: 10 + start_period: 20s restart: unless-stopped frontend: @@ -88,19 +96,31 @@ services: condition: service_started auth: condition: service_started + healthcheck: + test: ["CMD-SHELL", "wget -qO- http://localhost/ >/dev/null 2>&1 || exit 1"] + interval: 15s + timeout: 5s + retries: 10 restart: unless-stopped nginx: image: nginx:1.27-alpine depends_on: - - backend - - auth - - frontend - - mailpit + backend: + condition: service_started + auth: + condition: service_started + frontend: + condition: service_started ports: - "${NGINX_PORT}:80" volumes: - ./infra/nginx/default.conf:/etc/nginx/conf.d/default.conf:ro + healthcheck: + test: ["CMD-SHELL", "wget -qO- http://localhost/nginx-health >/dev/null 2>&1 || exit 1"] + interval: 15s + timeout: 5s + retries: 10 restart: unless-stopped volumes: diff --git a/infra/nginx/default.conf b/infra/nginx/default.conf index 2ebb8e4..cdac209 100644 --- a/infra/nginx/default.conf +++ b/infra/nginx/default.conf @@ -9,6 +9,16 @@ server { # startup-time resolution forever (stale upstream → 502). resolver 127.0.0.11 valid=10s ipv6=off; + # Baseline security headers on every response, API included. + add_header X-Frame-Options "DENY" always; + add_header X-Content-Type-Options "nosniff" always; + add_header Referrer-Policy "strict-origin-when-cross-origin" always; + add_header Permissions-Policy "accelerometer=(), camera=(), geolocation=(), gyroscope=(), microphone=(), usb=()" always; + + location = /nginx-health { + return 204; + } + location /api/ { set $upstream http://backend:8080; proxy_pass $upstream; @@ -40,18 +50,19 @@ server { proxy_set_header X-Forwarded-Proto $scheme; } - location /mailpit/ { - set $upstream http://mailpit:8025; - rewrite ^/mailpit/?(.*)$ /$1 break; - proxy_pass $upstream; - proxy_http_version 1.1; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $scheme; - } + # Dev-only location blocks (mailpit) are dropped in here by + # docker-compose.dev.yml. A glob with no matches is legal, so production + # simply includes nothing. + include /etc/nginx/primora-extra/*.conf; location / { + # Any add_header inside a location replaces ALL inherited add_headers — + # so the baseline four must be repeated here next to the SPA-only CSP. + add_header X-Frame-Options "DENY" always; + add_header X-Content-Type-Options "nosniff" always; + add_header Referrer-Policy "strict-origin-when-cross-origin" always; + add_header Permissions-Policy "accelerometer=(), camera=(), geolocation=(), gyroscope=(), microphone=(), usb=()" always; + add_header Content-Security-Policy "default-src 'self'; script-src 'self'; style-src 'self' 'unsafe-inline'; img-src 'self' data: blob:; font-src 'self' data:; connect-src 'self'; worker-src 'self'; manifest-src 'self'; object-src 'none'; frame-ancestors 'none'; base-uri 'self'; form-action 'self'" always; set $upstream http://frontend:80; proxy_pass $upstream; proxy_http_version 1.1; diff --git a/infra/nginx/mailpit.conf b/infra/nginx/mailpit.conf new file mode 100644 index 0000000..4538ee0 --- /dev/null +++ b/infra/nginx/mailpit.conf @@ -0,0 +1,12 @@ +# Dev-only: mounted into /etc/nginx/primora-extra/ by docker-compose.dev.yml. +# Included inside the default server block — do not add a server{} here. +location /mailpit/ { + set $upstream http://mailpit:8025; + rewrite ^/mailpit/?(.*)$ /$1 break; + proxy_pass $upstream; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; +} diff --git a/packages/api-client/src/generated/index.ts b/packages/api-client/src/generated/index.ts index 07fc527..9fb4e2d 100644 --- a/packages/api-client/src/generated/index.ts +++ b/packages/api-client/src/generated/index.ts @@ -55,6 +55,8 @@ export type { DBRedisRequest } from './models/DBRedisRequest'; export type { DBRedisResponse } from './models/DBRedisResponse'; export type { DBSchemaContextResponse } from './models/DBSchemaContextResponse'; export type { DBTableListResponse } from './models/DBTableListResponse'; +export type { DBTransferRequest } from './models/DBTransferRequest'; +export type { DBTransferResponse } from './models/DBTransferResponse'; export type { DBXStatus } from './models/DBXStatus'; export type { DeployMarker } from './models/DeployMarker'; export type { Document } from './models/Document'; @@ -168,6 +170,8 @@ export { $DBRedisRequest } from './schemas/$DBRedisRequest'; export { $DBRedisResponse } from './schemas/$DBRedisResponse'; export { $DBSchemaContextResponse } from './schemas/$DBSchemaContextResponse'; export { $DBTableListResponse } from './schemas/$DBTableListResponse'; +export { $DBTransferRequest } from './schemas/$DBTransferRequest'; +export { $DBTransferResponse } from './schemas/$DBTransferResponse'; export { $DBXStatus } from './schemas/$DBXStatus'; export { $DeployMarker } from './schemas/$DeployMarker'; export { $Document } from './schemas/$Document'; diff --git a/packages/api-client/src/generated/models/ApiKey.ts b/packages/api-client/src/generated/models/ApiKey.ts index 7530a02..159c89a 100644 --- a/packages/api-client/src/generated/models/ApiKey.ts +++ b/packages/api-client/src/generated/models/ApiKey.ts @@ -7,6 +7,7 @@ export type ApiKey = { project_id: string; name: string; prefix: string; + scopes?: Array<'ingest' | 'read' | 'write' | 'admin'>; last_used_at?: string | null; revoked_at?: string | null; }; diff --git a/packages/api-client/src/generated/models/CreateApiKeyRequest.ts b/packages/api-client/src/generated/models/CreateApiKeyRequest.ts index ac76a42..980311d 100644 --- a/packages/api-client/src/generated/models/CreateApiKeyRequest.ts +++ b/packages/api-client/src/generated/models/CreateApiKeyRequest.ts @@ -4,5 +4,9 @@ /* eslint-disable */ export type CreateApiKeyRequest = { name: string; + /** + * Access scopes for the key. admin unlocks everything; write unlocks read + ingest + non-admin mutations; read and ingest unlock only themselves. Omitted means ["admin"] (full access, legacy behaviour). + */ + scopes?: Array<'ingest' | 'read' | 'write' | 'admin'>; }; diff --git a/packages/api-client/src/generated/models/CreatedApiKey.ts b/packages/api-client/src/generated/models/CreatedApiKey.ts index 2c3ffb4..af47c8b 100644 --- a/packages/api-client/src/generated/models/CreatedApiKey.ts +++ b/packages/api-client/src/generated/models/CreatedApiKey.ts @@ -7,5 +7,6 @@ export type CreatedApiKey = { prefix: string; secret: string; name: string; + scopes?: Array<'ingest' | 'read' | 'write' | 'admin'>; }; diff --git a/packages/api-client/src/generated/models/DBTransferRequest.ts b/packages/api-client/src/generated/models/DBTransferRequest.ts new file mode 100644 index 0000000..72f9d05 --- /dev/null +++ b/packages/api-client/src/generated/models/DBTransferRequest.ts @@ -0,0 +1,30 @@ +/* generated using openapi-typescript-codegen -- do not edit */ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ +export type DBTransferRequest = { + target_connection_id: string; + /** + * SELECT run against the source connection. + */ + query: string; + database?: string; + /** + * Destination table for SQL targets (schema-qualified allowed). + */ + target_table?: string; + /** + * Redis targets: SET key template with {column} placeholders, e.g. "user:{id}". + */ + key_pattern?: string; + /** + * Redis targets: column whose cell becomes the value. Omitted stores the whole row as JSON. + */ + value_column?: string; + /** + * Optional EX TTL applied to each Redis SET. + */ + ttl_seconds?: number; + limit?: number; +}; + diff --git a/packages/api-client/src/generated/models/DBTransferResponse.ts b/packages/api-client/src/generated/models/DBTransferResponse.ts new file mode 100644 index 0000000..39cbb7f --- /dev/null +++ b/packages/api-client/src/generated/models/DBTransferResponse.ts @@ -0,0 +1,11 @@ +/* generated using openapi-typescript-codegen -- do not edit */ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ +export type DBTransferResponse = { + transferred: number; + truncated: boolean; + source: string; + target: string; +}; + diff --git a/packages/api-client/src/generated/models/Project.ts b/packages/api-client/src/generated/models/Project.ts index d220b64..1a00add 100644 --- a/packages/api-client/src/generated/models/Project.ts +++ b/packages/api-client/src/generated/models/Project.ts @@ -9,5 +9,8 @@ export type Project = { name: string; description?: string | null; membership_role?: string | null; + retention_events_days?: number; + retention_audit_days?: number; + retention_webhook_days?: number; }; diff --git a/packages/api-client/src/generated/models/UpdateProjectRequest.ts b/packages/api-client/src/generated/models/UpdateProjectRequest.ts index 13e7cb6..068bea0 100644 --- a/packages/api-client/src/generated/models/UpdateProjectRequest.ts +++ b/packages/api-client/src/generated/models/UpdateProjectRequest.ts @@ -6,5 +6,17 @@ export type UpdateProjectRequest = { name: string; slug: string; description?: string | null; + /** + * Telemetry event retention; 0 disables the sweep. + */ + retention_events_days?: number; + /** + * Audit log retention; 0 disables the sweep. + */ + retention_audit_days?: number; + /** + * Webhook delivery retention; 0 disables the sweep. + */ + retention_webhook_days?: number; }; diff --git a/packages/api-client/src/generated/schemas/$ApiKey.ts b/packages/api-client/src/generated/schemas/$ApiKey.ts index 78eef49..2e1ed7a 100644 --- a/packages/api-client/src/generated/schemas/$ApiKey.ts +++ b/packages/api-client/src/generated/schemas/$ApiKey.ts @@ -22,6 +22,12 @@ export const $ApiKey = { type: 'string', isRequired: true, }, + scopes: { + type: 'array', + contains: { + type: 'Enum', + }, + }, last_used_at: { type: 'string', isNullable: true, diff --git a/packages/api-client/src/generated/schemas/$CreateApiKeyRequest.ts b/packages/api-client/src/generated/schemas/$CreateApiKeyRequest.ts index beb98f1..483ecae 100644 --- a/packages/api-client/src/generated/schemas/$CreateApiKeyRequest.ts +++ b/packages/api-client/src/generated/schemas/$CreateApiKeyRequest.ts @@ -8,5 +8,11 @@ export const $CreateApiKeyRequest = { type: 'string', isRequired: true, }, + scopes: { + type: 'array', + contains: { + type: 'Enum', + }, + }, }, } as const; diff --git a/packages/api-client/src/generated/schemas/$CreatedApiKey.ts b/packages/api-client/src/generated/schemas/$CreatedApiKey.ts index bb6036b..7f54120 100644 --- a/packages/api-client/src/generated/schemas/$CreatedApiKey.ts +++ b/packages/api-client/src/generated/schemas/$CreatedApiKey.ts @@ -21,5 +21,11 @@ export const $CreatedApiKey = { type: 'string', isRequired: true, }, + scopes: { + type: 'array', + contains: { + type: 'Enum', + }, + }, }, } as const; diff --git a/packages/api-client/src/generated/schemas/$DBTransferRequest.ts b/packages/api-client/src/generated/schemas/$DBTransferRequest.ts new file mode 100644 index 0000000..4a8acbf --- /dev/null +++ b/packages/api-client/src/generated/schemas/$DBTransferRequest.ts @@ -0,0 +1,42 @@ +/* generated using openapi-typescript-codegen -- do not edit */ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ +export const $DBTransferRequest = { + properties: { + target_connection_id: { + type: 'string', + isRequired: true, + format: 'uuid', + }, + query: { + type: 'string', + description: `SELECT run against the source connection.`, + isRequired: true, + }, + database: { + type: 'string', + }, + target_table: { + type: 'string', + description: `Destination table for SQL targets (schema-qualified allowed).`, + }, + key_pattern: { + type: 'string', + description: `Redis targets: SET key template with {column} placeholders, e.g. "user:{id}".`, + }, + value_column: { + type: 'string', + description: `Redis targets: column whose cell becomes the value. Omitted stores the whole row as JSON.`, + }, + ttl_seconds: { + type: 'number', + description: `Optional EX TTL applied to each Redis SET.`, + }, + limit: { + type: 'number', + maximum: 1000, + minimum: 1, + }, + }, +} as const; diff --git a/packages/api-client/src/generated/schemas/$DBTransferResponse.ts b/packages/api-client/src/generated/schemas/$DBTransferResponse.ts new file mode 100644 index 0000000..4a38b14 --- /dev/null +++ b/packages/api-client/src/generated/schemas/$DBTransferResponse.ts @@ -0,0 +1,24 @@ +/* generated using openapi-typescript-codegen -- do not edit */ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ +export const $DBTransferResponse = { + properties: { + transferred: { + type: 'number', + isRequired: true, + }, + truncated: { + type: 'boolean', + isRequired: true, + }, + source: { + type: 'string', + isRequired: true, + }, + target: { + type: 'string', + isRequired: true, + }, + }, +} as const; diff --git a/packages/api-client/src/generated/schemas/$Project.ts b/packages/api-client/src/generated/schemas/$Project.ts index b8ced3f..31f267f 100644 --- a/packages/api-client/src/generated/schemas/$Project.ts +++ b/packages/api-client/src/generated/schemas/$Project.ts @@ -30,5 +30,14 @@ export const $Project = { type: 'string', isNullable: true, }, + retention_events_days: { + type: 'number', + }, + retention_audit_days: { + type: 'number', + }, + retention_webhook_days: { + type: 'number', + }, }, } as const; diff --git a/packages/api-client/src/generated/schemas/$UpdateProjectRequest.ts b/packages/api-client/src/generated/schemas/$UpdateProjectRequest.ts index 0ad9aab..7a38c56 100644 --- a/packages/api-client/src/generated/schemas/$UpdateProjectRequest.ts +++ b/packages/api-client/src/generated/schemas/$UpdateProjectRequest.ts @@ -16,5 +16,20 @@ export const $UpdateProjectRequest = { type: 'string', isNullable: true, }, + retention_events_days: { + type: 'number', + description: `Telemetry event retention; 0 disables the sweep.`, + maximum: 3650, + }, + retention_audit_days: { + type: 'number', + description: `Audit log retention; 0 disables the sweep.`, + maximum: 3650, + }, + retention_webhook_days: { + type: 'number', + description: `Webhook delivery retention; 0 disables the sweep.`, + maximum: 3650, + }, }, } as const; diff --git a/packages/api-client/src/generated/services/DatabasesService.ts b/packages/api-client/src/generated/services/DatabasesService.ts index b908ab0..1c8bf22 100644 --- a/packages/api-client/src/generated/services/DatabasesService.ts +++ b/packages/api-client/src/generated/services/DatabasesService.ts @@ -15,6 +15,8 @@ import type { DBRedisRequest } from '../models/DBRedisRequest'; import type { DBRedisResponse } from '../models/DBRedisResponse'; import type { DBSchemaContextResponse } from '../models/DBSchemaContextResponse'; import type { DBTableListResponse } from '../models/DBTableListResponse'; +import type { DBTransferRequest } from '../models/DBTransferRequest'; +import type { DBTransferResponse } from '../models/DBTransferResponse'; import type { DBXStatus } from '../models/DBXStatus'; import type { CancelablePromise } from '../core/CancelablePromise'; import { OpenAPI } from '../core/OpenAPI'; @@ -333,4 +335,33 @@ export class DatabasesService { }, }); } + /** + * Move query results from this connection to another saved connection + * @returns DBTransferResponse Transfer result + * @throws ApiError + */ + public static transferDbRows({ + projectId, + connectionId, + requestBody, + }: { + projectId: string, + connectionId: string, + requestBody: DBTransferRequest, + }): CancelablePromise { + return __request(OpenAPI, { + method: 'POST', + url: '/projects/{projectID}/db-connections/{connectionID}/transfer', + path: { + 'projectID': projectId, + 'connectionID': connectionId, + }, + body: requestBody, + mediaType: 'application/json', + errors: { + 404: `Error response`, + 502: `Error response`, + }, + }); + } } diff --git a/packages/api-client/src/generated/services/HealthService.ts b/packages/api-client/src/generated/services/HealthService.ts index ada2aad..2dddc60 100644 --- a/packages/api-client/src/generated/services/HealthService.ts +++ b/packages/api-client/src/generated/services/HealthService.ts @@ -28,4 +28,15 @@ export class HealthService { url: '/health/readiness', }); } + /** + * Prometheus text exposition of platform HTTP metrics + * @returns string Prometheus metrics + * @throws ApiError + */ + public static getMetrics(): CancelablePromise { + return __request(OpenAPI, { + method: 'GET', + url: '/metrics', + }); + } } diff --git a/scripts/backup.sh b/scripts/backup.sh new file mode 100755 index 0000000..502c7aa --- /dev/null +++ b/scripts/backup.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash +# Primora backup: pg_dump of the database + archive of the object-storage root. +# +# Produces backups/primora-YYYYMMDD-HHMMSS.tar.gz containing: +# db.dump postgres custom-format dump (all schemas, incl. core + auth) +# storage/ contents of the backend_storage volume +# +# Usage (from the repo root, against the compose stack): +# ./scripts/backup.sh +# +# Optional env: +# BACKUP_DIR output directory (default: ./backups) +# POSTGRES_CONTAINER compose container name (default: primora-postgres-1) +# POSTGRES_USER/DB dump credentials (default: primora/primora) +# STORAGE_VOLUME docker volume for objects (default: primora_backend_storage) +# +# Restore with scripts/restore.sh. Schedule via cron or a systemd timer — +# see DEPLOYMENT_GUIDE.md#backups. + +set -euo pipefail + +BACKUP_DIR="${BACKUP_DIR:-./backups}" +POSTGRES_CONTAINER="${POSTGRES_CONTAINER:-primora-postgres-1}" +POSTGRES_USER="${POSTGRES_USER:-primora}" +POSTGRES_DB="${POSTGRES_DB:-primora}" +STORAGE_VOLUME="${STORAGE_VOLUME:-primora_backend_storage}" + +stamp="$(date +%Y%m%d-%H%M%S)" +mkdir -p "$BACKUP_DIR" +BACKUP_DIR="$(cd "$BACKUP_DIR" && pwd)" +# Workdir lives under BACKUP_DIR — /tmp is not bind-mountable on every +# docker setup, and the storage archive is written through a container. +work="$(mktemp -d "$BACKUP_DIR/.tmp.XXXXXX")" +out="$BACKUP_DIR/primora-$stamp.tar.gz" +trap 'rm -rf "$work"' EXIT + +echo "==> dumping database from $POSTGRES_CONTAINER" +docker exec "$POSTGRES_CONTAINER" \ + pg_dump -U "$POSTGRES_USER" -d "$POSTGRES_DB" -Fc \ + > "$work/db.dump" +[ -s "$work/db.dump" ] || { echo "pg_dump produced an empty file" >&2; exit 1; } + +echo "==> archiving storage volume $STORAGE_VOLUME" +# Read the volume through a throwaway container — no host mount assumptions. +docker run --rm -v "$STORAGE_VOLUME:/data:ro" -v "$work:/backup" \ + alpine:3 tar -C /data -cf /backup/storage.tar . +[ -s "$work/storage.tar" ] || echo "warning: storage volume appears empty" >&2 + +tar -C "$work" -czf "$out" db.dump storage.tar +echo "==> wrote $out ($(du -h "$out" | cut -f1))" diff --git a/scripts/load-sanity.js b/scripts/load-sanity.js new file mode 100644 index 0000000..4224527 --- /dev/null +++ b/scripts/load-sanity.js @@ -0,0 +1,75 @@ +// Primora load sanity — NOT a benchmark. Verifies the ingest path and a hot +// read endpoint hold up under modest concurrency without errors. +// +// k6 run scripts/load-sanity.js \ +// -e PRIMORA_API_KEY=prm_xxx_xxx \ +// -e PRIMORA_PROJECT_ID= \ +// -e BASE_URL=http://localhost +// +// The key needs the ingest + read scopes. Create one in Settings → API keys. + +import http from "k6/http"; +import { check } from "k6"; + +const BASE = __ENV.BASE_URL || "http://localhost"; +const API_KEY = __ENV.PRIMORA_API_KEY || ""; +const PROJECT = __ENV.PRIMORA_PROJECT_ID || ""; + +export const options = { + scenarios: { + ingest: { + exec: "ingest", + executor: "constant-vus", + vus: 5, + duration: "30s", + }, + hot_read: { + exec: "hotRead", + executor: "constant-vus", + vus: 10, + duration: "30s", + }, + }, + // 429s are the rate limiter doing its job (API_KEY_RATE_LIMIT_PER_MINUTE, + // default 600/min) — counted as expected via responseCallback below. The + // sanity bar is: no real errors, no hangs. Raise the env limit or lower + // VUs to exercise unthrottled throughput. + thresholds: { + http_req_failed: ["rate<0.01"], + http_req_duration: ["p(95)<500"], + }, +}; + +const ingestBody = JSON.stringify({ + events: [ + { + type: "metric", + severity: "info", + component: "load-sanity", + message: "k6 sanity metric", + payload: { name: "k6.sanity", value: 1 }, + }, + ], +}); + +export function ingest() { + const res = http.post(`${BASE}/api/v1/ingest`, ingestBody, { + headers: { + "content-type": "application/json", + "X-Primora-Key": API_KEY, + }, + responseCallback: http.expectedStatuses(202, 429), + }); + check(res, { "ingest 202 or throttled": (r) => r.status === 202 || r.status === 429 }); +} + +export function hotRead() { + const res = http.get( + `${BASE}/api/v1/projects/${PROJECT}/events?limit=25`, + { + headers: { "X-Primora-Key": API_KEY }, + responseCallback: http.expectedStatuses(200, 429), + }, + ); + check(res, { "events 200 or throttled": (r) => r.status === 200 || r.status === 429 }); +} diff --git a/scripts/restore.sh b/scripts/restore.sh new file mode 100755 index 0000000..30107f9 --- /dev/null +++ b/scripts/restore.sh @@ -0,0 +1,47 @@ +#!/usr/bin/env bash +# Primora restore: rebuilds the database and storage volume from a backup +# tarball produced by scripts/backup.sh. +# +# ./scripts/restore.sh backups/primora-20260101-120000.tar.gz +# +# DESTRUCTIVE: drops and recreates the target database's objects and +# overwrites the storage volume. Stop the stack first: +# docker compose stop backend auth +# Then restore, then `docker compose up -d`. + +set -euo pipefail + +ARCHIVE="${1:?usage: restore.sh }" +POSTGRES_CONTAINER="${POSTGRES_CONTAINER:-primora-postgres-1}" +POSTGRES_USER="${POSTGRES_USER:-primora}" +POSTGRES_DB="${POSTGRES_DB:-primora}" +STORAGE_VOLUME="${STORAGE_VOLUME:-primora_backend_storage}" + +[ -f "$ARCHIVE" ] || { echo "no such archive: $ARCHIVE" >&2; exit 1; } +docker exec "$POSTGRES_CONTAINER" true 2>/dev/null \ + || { echo "postgres container $POSTGRES_CONTAINER is not running" >&2; exit 1; } + +echo "This will overwrite database '$POSTGRES_DB' in $POSTGRES_CONTAINER" +echo "and the contents of volume $STORAGE_VOLUME." +read -r -p "Continue? [y/N] " confirm +[ "$confirm" = "y" ] || [ "$confirm" = "Y" ] || { echo "aborted"; exit 1; } + +# Workdir under cwd — /tmp is not bind-mountable on every docker setup. +work="$(mktemp -d "$(pwd)/.restore-tmp.XXXXXX")" +trap 'rm -rf "$work"' EXIT +tar -xzf "$ARCHIVE" -C "$work" +[ -f "$work/db.dump" ] || { echo "archive has no db.dump" >&2; exit 1; } + +echo "==> restoring database" +cat "$work/db.dump" | docker exec -i "$POSTGRES_CONTAINER" \ + pg_restore -U "$POSTGRES_USER" -d "$POSTGRES_DB" --clean --if-exists --no-owner + +if [ -f "$work/storage.tar" ]; then + echo "==> restoring storage volume $STORAGE_VOLUME" + docker run --rm -v "$STORAGE_VOLUME:/data" -v "$work:/backup" \ + alpine:3 sh -c 'rm -rf /data/* /data/..?* /data/.[!.]* 2>/dev/null; tar -C /data -xf /backup/storage.tar' +else + echo "warning: archive has no storage.tar — skipping volume" >&2 +fi + +echo "==> restore complete; restart the stack with: docker compose up -d" diff --git a/scripts/setup.sh b/scripts/setup.sh index 818cc71..9c90517 100755 --- a/scripts/setup.sh +++ b/scripts/setup.sh @@ -26,15 +26,18 @@ if [ ! -f .env ]; then echo "🔐 Generating secure secrets..." JWT_SECRET=$(openssl rand -base64 32) BETTER_AUTH_SECRET=$(openssl rand -base64 32) + ENCRYPTION_KEY=$(openssl rand -hex 32) # Update .env with secrets # Use different sed approach for better compatibility if [[ "$OSTYPE" == "darwin"* ]]; then sed -i '' "s/JWT_SECRET=change-me-super-long-jwt-secret/JWT_SECRET=$JWT_SECRET/g" .env sed -i '' "s/BETTER_AUTH_SECRET=change-me-super-long-better-auth-secret/BETTER_AUTH_SECRET=$BETTER_AUTH_SECRET/g" .env + sed -i '' "s/^PRIMORA_ENCRYPTION_KEY=.*/PRIMORA_ENCRYPTION_KEY=$ENCRYPTION_KEY/" .env else sed -i "s/JWT_SECRET=change-me-super-long-jwt-secret/JWT_SECRET=$JWT_SECRET/g" .env sed -i "s/BETTER_AUTH_SECRET=change-me-super-long-better-auth-secret/BETTER_AUTH_SECRET=$BETTER_AUTH_SECRET/g" .env + sed -i "s/^PRIMORA_ENCRYPTION_KEY=.*/PRIMORA_ENCRYPTION_KEY=$ENCRYPTION_KEY/" .env fi echo "✅ .env file created and secured." else @@ -55,12 +58,13 @@ else sed -i "s/VITE_APP_URL=http:\/\/localhost/VITE_APP_URL=http:\/\/$DOMAIN/g" .env fi -# Start services +# Start services — the dev overlay adds mailpit for local email capture. +# Production deploys use the base file alone: docker compose up -d echo "📦 Pulling images and starting services..." if command -v docker-compose &> /dev/null; then - docker-compose up -d + docker-compose -f docker-compose.yml -f docker-compose.dev.yml up -d else - docker compose up -d + docker compose -f docker-compose.yml -f docker-compose.dev.yml up -d fi echo "" diff --git a/scripts/smoke-e2e.sh b/scripts/smoke-e2e.sh new file mode 100755 index 0000000..389b43a --- /dev/null +++ b/scripts/smoke-e2e.sh @@ -0,0 +1,102 @@ +#!/usr/bin/env bash +# End-to-end smoke test against a running Primora stack. +# Expects the compose stack on BASE_URL (nginx :80 by default). +# +# ./scripts/smoke-e2e.sh +# BASE_URL=http://localhost:8080 ./scripts/smoke-e2e.sh +# +# Covers: health, signup/login, bootstrap, scoped API keys, ingest, +# telemetry read, and the Prometheus endpoint. Exits non-zero on failure. + +set -euo pipefail + +BASE="${BASE_URL:-http://localhost}" +EMAIL="smoke-$(date +%s)@example.com" +PASSWORD="smoke-password-$(date +%s)" +JAR="$(mktemp)" +trap 'rm -f "$JAR"' EXIT + +fail() { echo "FAIL: $*" >&2; exit 1; } +step() { echo "==> $*"; } + +wait_for() { + local url="$1" name="$2" tries=60 + while (( tries-- > 0 )); do + if curl -fsS -o /dev/null "$url" 2>/dev/null; then + echo " $name ready" + return 0 + fi + sleep 2 + done + fail "$name did not become ready ($url)" +} + +step "waiting for services" +wait_for "$BASE/api/v1/health/liveness" "backend liveness" +wait_for "$BASE/api/v1/health/readiness" "backend readiness" +wait_for "$BASE/auth-health" "auth" + +step "signup $EMAIL" +code=$(curl -sS -o /dev/null -w '%{http_code}' -c "$JAR" \ + -H 'content-type: application/json' \ + -d "{\"email\":\"$EMAIL\",\"password\":\"$PASSWORD\",\"name\":\"Smoke Test\"}" \ + "$BASE/auth/sign-up/email") +[ "$code" = "200" ] || [ "$code" = "201" ] || fail "sign-up returned $code" + +step "fetch api token" +token=$(curl -fsS -b "$JAR" "$BASE/auth/token" | sed -n 's/.*"token":"\([^"]*\)".*/\1/p') +[ -n "$token" ] || fail "no token in /auth/token response" + +step "bootstrap organization + project" +code=$(curl -sS -o /tmp/smoke-bootstrap.json -w '%{http_code}' -H "authorization: Bearer $token" -H 'content-type: application/json' \ + -d '{"organizationName":"Smoke Org","organizationSlug":"smoke-org","projectName":"Smoke Project","projectSlug":"smoke-project"}' \ + "$BASE/api/v1/bootstrap") +if [ "$code" = "409" ]; then + # Instance already bootstrapped — create a fresh org + project for this user. + org_id=$(curl -fsS -H "authorization: Bearer $token" -H 'content-type: application/json' \ + -d "{\"name\":\"Smoke Org $(date +%s)\",\"slug\":\"smoke-$(date +%s)\"}" \ + "$BASE/api/v1/organizations" | sed -n 's/.*"id":"\([^"]*\)".*/\1/p') + [ -n "$org_id" ] || fail "could not create organization" + project_id=$(curl -fsS -H "authorization: Bearer $token" -H 'content-type: application/json' \ + -d "{\"name\":\"Smoke Project\",\"slug\":\"smoke-$(date +%s)-p\"}" \ + "$BASE/api/v1/organizations/$org_id/projects" | sed -n 's/.*"id":"\([^"]*\)".*/\1/p') +else + [ "$code" = "200" ] || [ "$code" = "201" ] || { cat /tmp/smoke-bootstrap.json >&2; fail "bootstrap returned $code"; } + project_id=$(sed -n 's/.*"project_id":"\([^"]*\)".*/\1/p' /tmp/smoke-bootstrap.json) +fi +[ -n "$project_id" ] || fail "no project_id available" +echo " project $project_id" + +step "create ingest+read API key" +key_json=$(curl -fsS -H "authorization: Bearer $token" -H 'content-type: application/json' \ + -d '{"name":"smoke-ingest","scopes":["ingest","read"]}' \ + "$BASE/api/v1/projects/$project_id/api-keys") +api_key=$(echo "$key_json" | sed -n 's/.*"secret":"\([^"]*\)".*/\1/p') +[ -n "$api_key" ] || fail "no secret in api key response" +echo "$key_json" | grep -q '"ingest"' || fail "scopes missing from api key response" + +step "ingest event via X-Primora-Key" +curl -fsS -o /dev/null -H "X-Primora-Key: $api_key" -H 'content-type: application/json' \ + -d '{"events":[{"type":"metric","severity":"info","component":"smoke","message":"smoke metric","payload":{"name":"smoke.count","value":1}}]}' \ + "$BASE/api/v1/ingest" + +step "negative: read-only key cannot ingest" +ro_json=$(curl -fsS -H "authorization: Bearer $token" -H 'content-type: application/json' \ + -d '{"name":"smoke-readonly","scopes":["read"]}' \ + "$BASE/api/v1/projects/$project_id/api-keys") +ro_key=$(echo "$ro_json" | sed -n 's/.*"secret":"\([^"]*\)".*/\1/p') +code=$(curl -sS -o /dev/null -w '%{http_code}' -H "X-Primora-Key: $ro_key" -H 'content-type: application/json' \ + -d '{"type":"metric","severity":"info","message":"should be rejected"}' \ + "$BASE/api/v1/ingest") +[ "$code" = "403" ] || fail "read-only key ingest returned $code, expected 403" + +step "read back events" +curl -fsS -o /dev/null -H "authorization: Bearer $token" \ + "$BASE/api/v1/projects/$project_id/events?limit=5" + +step "prometheus metrics" +curl -fsS "$BASE/api/v1/metrics" | grep -q 'primora_http_requests_total' \ + || fail "metrics endpoint missing primora_http_requests_total" + +echo +echo "smoke: all checks passed" diff --git a/scripts/verify-production-ready.sh b/scripts/verify-production-ready.sh index 91d6540..babe76c 100755 --- a/scripts/verify-production-ready.sh +++ b/scripts/verify-production-ready.sh @@ -20,20 +20,20 @@ FAILED=0 check_file() { if [ -f "$1" ]; then echo -e "${GREEN}✓${NC} $2" - ((PASSED++)) + ((++PASSED)) else echo -e "${RED}✗${NC} $2 (missing: $1)" - ((FAILED++)) + ((++FAILED)) fi } check_dir() { if [ -d "$1" ]; then echo -e "${GREEN}✓${NC} $2" - ((PASSED++)) + ((++PASSED)) else echo -e "${RED}✗${NC} $2 (missing: $1)" - ((FAILED++)) + ((++FAILED)) fi } @@ -58,8 +58,18 @@ echo "" echo "📚 Documentation:" check_file "PRODUCTION_READINESS.md" "Production readiness checklist" check_file "DEPLOYMENT_GUIDE.md" "Deployment guide" -check_file "PRODUCTION_IMPROVEMENTS.md" "Improvements summary" -check_file "PRODUCTION_READY_SUMMARY.md" "Production ready summary" +check_file "ROADMAP.md" "Roadmap" +echo "" + +echo "🛡️ Phase 6 hardening:" +check_file "apps/backend/db/migrations/00006_api_key_scopes.sql" "API key scopes migration" +check_file "apps/backend/db/migrations/00007_retention.sql" "Retention migration" +check_file "apps/backend/internal/services/retention.go" "Retention sweeper" +check_file "docker-compose.dev.yml" "Dev overlay (Mailpit isolation)" +check_file "scripts/smoke-e2e.sh" "E2E smoke script" +check_file "scripts/backup.sh" "Backup script" +check_file "scripts/restore.sh" "Restore script" +check_file "scripts/load-sanity.js" "k6 load sanity script" echo "" echo "🐳 Infrastructure:"