mirror of
https://github.com/Dvorinka/Primora.git
synced 2026-10-28 07:41:23 +00:00
feat: phase 6 production hardening + DB-to-DB transfer
Security:
- Graceful shutdown: http.Server + signal.NotifyContext, 10s drain,
retention sweeper cancelled before pool close
- Scoped API keys (migration 00006): ingest/read/write/admin enforced in
auth middleware; scope picker + badges in Settings UI; scopeless legacy
keys act as admin. Read-only keys now get 403 on ingest (verified live)
- nginx: baseline security headers on all responses, CSP scoped to SPA;
fixed add_header inheritance shadowing
- better-auth rate limit enabled unconditionally (was gated on NODE_ENV,
which the shipped .env sets to development)
- Mailpit moved to docker-compose.dev.yml; prod compose cannot expose it
- setup.sh generates PRIMORA_ENCRYPTION_KEY
Reliability/ops:
- Retention: per-project retention_{events,audit,webhook}_days
(migration 00007, 0=off) + hourly RetentionSweeper; edit UI in Projects
- GET /api/v1/metrics Prometheus exposition
- compose: healthchecks + restart policies everywhere, dragonfly pinned
to v2.0.0, depends_on service_healthy
- backup.sh/restore.sh (pg_dump -Fc + storage volume); restore verified
against scratch postgres+volume
- .dockerignore: exclude target dirs (build context was 4.7GB)
Correctness/testing:
- DBX transfer endpoint: POST /db-connections/:id/transfer — SQL or Redis
target, 1000-row bound, server-side creds (verified live pg→dragonfly)
- Pagination audit: ListComponents capped at 500 (auto-registers)
- CI: goose up/reset/up job + compose e2e-smoke job (smoke-e2e.sh
verified locally: signup, bootstrap, scoped keys, ingest, metrics)
- k6 load sanity script: 6650 reqs, ~217/s, 0 real failures, p95 221ms
(429s = configured rate limiter working)
Docs: DEPLOYMENT_GUIDE.md, PRODUCTION_READINESS.md; verify script
repaired (stale refs + set -e arithmetic abort). ROADMAP: phase 2 +
phase 6 marked complete.
Authored By: TDvorak <[email protected]>
Generated with [Devin](https://devin.ai)
Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
1 parent
1b34dd43bb
commit
55edbf8b56
64 files changed
+2058
-165
No files matched your search
@@ -2,12 +2,16 @@ package app
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"os/signal"
|
||||
"strconv"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
@@ -37,6 +41,8 @@ type App struct {
|
||||
Redis *redis.Client
|
||||
DBX *dbx.Client
|
||||
Platform *services.PlatformService
|
||||
|
||||
sweepCancel context.CancelFunc
|
||||
}
|
||||
|
||||
func Bootstrap(ctx context.Context) (*App, error) {
|
||||
@@ -157,21 +163,53 @@ func Bootstrap(ctx context.Context) (*App, error) {
|
||||
}
|
||||
handler.Register(router)
|
||||
|
||||
sweepCtx, sweepCancel := context.WithCancel(context.Background())
|
||||
sweeper := services.NewRetentionSweeper(repo, logger)
|
||||
go sweeper.Run(sweepCtx)
|
||||
|
||||
return &App{
|
||||
Config: cfg,
|
||||
Router: router,
|
||||
Logger: logger,
|
||||
DB: dbPool,
|
||||
Redis: redisClient,
|
||||
DBX: dbxClient,
|
||||
Platform: platform,
|
||||
Config: cfg,
|
||||
Router: router,
|
||||
Logger: logger,
|
||||
DB: dbPool,
|
||||
Redis: redisClient,
|
||||
DBX: dbxClient,
|
||||
Platform: platform,
|
||||
sweepCancel: sweepCancel,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (a *App) Run() error {
|
||||
address := ":" + a.Config.ServerPort
|
||||
a.Logger.Info("primora backend starting", "address", address)
|
||||
return a.Router.Run(address)
|
||||
srv := &http.Server{Addr: address, Handler: a.Router}
|
||||
|
||||
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
|
||||
defer stop()
|
||||
|
||||
errCh := make(chan error, 1)
|
||||
go func() {
|
||||
a.Logger.Info("primora backend starting", "address", address)
|
||||
errCh <- srv.ListenAndServe()
|
||||
}()
|
||||
|
||||
select {
|
||||
case err := <-errCh:
|
||||
return err
|
||||
case <-ctx.Done():
|
||||
}
|
||||
|
||||
a.Logger.Info("shutdown signal received, draining in-flight requests")
|
||||
shutdownCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
if err := srv.Shutdown(shutdownCtx); err != nil {
|
||||
return fmt.Errorf("graceful shutdown: %w", err)
|
||||
}
|
||||
if err := <-errCh; err != nil && !errors.Is(err, http.ErrServerClosed) {
|
||||
return err
|
||||
}
|
||||
// Stop the retention sweeper before main closes the DB pool.
|
||||
a.sweepCancel()
|
||||
return nil
|
||||
}
|
||||
|
||||
// firstPartyDBSeeds turns the platform's own Postgres and Dragonfly URLs into
|
||||
@@ -206,6 +244,9 @@ func firstPartyDBSeeds(cfg config.Config) []services.ManagedDBSeed {
|
||||
}
|
||||
|
||||
func (a *App) Close() error {
|
||||
if a.sweepCancel != nil {
|
||||
a.sweepCancel()
|
||||
}
|
||||
if a.Platform != nil {
|
||||
a.Platform.Close()
|
||||
}
|
||||
|
||||
@@ -18,9 +18,10 @@ INSERT INTO core.api_keys (
|
||||
name,
|
||||
prefix,
|
||||
secret_hash,
|
||||
scopes,
|
||||
created_by_user_id
|
||||
) VALUES ($1, $2, $3, $4, $5)
|
||||
RETURNING id, project_id, name, prefix, secret_hash, created_by_user_id, last_used_at, revoked_at, created_at
|
||||
) VALUES ($1, $2, $3, $4, $5, $6)
|
||||
RETURNING id, project_id, name, prefix, secret_hash, created_by_user_id, last_used_at, revoked_at, created_at, scopes
|
||||
`
|
||||
|
||||
type CreateAPIKeyParams struct {
|
||||
@@ -28,6 +29,7 @@ type CreateAPIKeyParams struct {
|
||||
Name string `json:"name"`
|
||||
Prefix string `json:"prefix"`
|
||||
SecretHash []byte `json:"secret_hash"`
|
||||
Scopes []string `json:"scopes"`
|
||||
CreatedByUserID pgtype.UUID `json:"created_by_user_id"`
|
||||
}
|
||||
|
||||
@@ -37,6 +39,7 @@ func (q *Queries) CreateAPIKey(ctx context.Context, arg CreateAPIKeyParams) (Cor
|
||||
arg.Name,
|
||||
arg.Prefix,
|
||||
arg.SecretHash,
|
||||
arg.Scopes,
|
||||
arg.CreatedByUserID,
|
||||
)
|
||||
var i CoreApiKey
|
||||
@@ -50,12 +53,13 @@ func (q *Queries) CreateAPIKey(ctx context.Context, arg CreateAPIKeyParams) (Cor
|
||||
&i.LastUsedAt,
|
||||
&i.RevokedAt,
|
||||
&i.CreatedAt,
|
||||
&i.Scopes,
|
||||
)
|
||||
return i, err
|
||||
}
|
||||
|
||||
const getAPIKeyByIDForProject = `-- name: GetAPIKeyByIDForProject :one
|
||||
SELECT id, project_id, name, prefix, secret_hash, created_by_user_id, last_used_at, revoked_at, created_at FROM core.api_keys
|
||||
SELECT id, project_id, name, prefix, secret_hash, created_by_user_id, last_used_at, revoked_at, created_at, scopes FROM core.api_keys
|
||||
WHERE project_id = $1
|
||||
AND id = $2
|
||||
`
|
||||
@@ -78,13 +82,14 @@ func (q *Queries) GetAPIKeyByIDForProject(ctx context.Context, arg GetAPIKeyByID
|
||||
&i.LastUsedAt,
|
||||
&i.RevokedAt,
|
||||
&i.CreatedAt,
|
||||
&i.Scopes,
|
||||
)
|
||||
return i, err
|
||||
}
|
||||
|
||||
const getAPIKeyByPrefix = `-- name: GetAPIKeyByPrefix :one
|
||||
SELECT
|
||||
ak.id, ak.project_id, ak.name, ak.prefix, ak.secret_hash, ak.created_by_user_id, ak.last_used_at, ak.revoked_at, ak.created_at,
|
||||
ak.id, ak.project_id, ak.name, ak.prefix, ak.secret_hash, ak.created_by_user_id, ak.last_used_at, ak.revoked_at, ak.created_at, ak.scopes,
|
||||
p.organization_id
|
||||
FROM core.api_keys ak
|
||||
JOIN core.projects p ON p.id = ak.project_id
|
||||
@@ -101,6 +106,7 @@ type GetAPIKeyByPrefixRow struct {
|
||||
LastUsedAt pgtype.Timestamptz `json:"last_used_at"`
|
||||
RevokedAt pgtype.Timestamptz `json:"revoked_at"`
|
||||
CreatedAt pgtype.Timestamptz `json:"created_at"`
|
||||
Scopes []string `json:"scopes"`
|
||||
OrganizationID uuid.UUID `json:"organization_id"`
|
||||
}
|
||||
|
||||
@@ -117,13 +123,14 @@ func (q *Queries) GetAPIKeyByPrefix(ctx context.Context, prefix string) (GetAPIK
|
||||
&i.LastUsedAt,
|
||||
&i.RevokedAt,
|
||||
&i.CreatedAt,
|
||||
&i.Scopes,
|
||||
&i.OrganizationID,
|
||||
)
|
||||
return i, err
|
||||
}
|
||||
|
||||
const listAPIKeysForProject = `-- name: ListAPIKeysForProject :many
|
||||
SELECT id, project_id, name, prefix, secret_hash, created_by_user_id, last_used_at, revoked_at, created_at FROM core.api_keys
|
||||
SELECT id, project_id, name, prefix, secret_hash, created_by_user_id, last_used_at, revoked_at, created_at, scopes FROM core.api_keys
|
||||
WHERE project_id = $1
|
||||
ORDER BY created_at DESC
|
||||
`
|
||||
@@ -147,6 +154,7 @@ func (q *Queries) ListAPIKeysForProject(ctx context.Context, projectID uuid.UUID
|
||||
&i.LastUsedAt,
|
||||
&i.RevokedAt,
|
||||
&i.CreatedAt,
|
||||
&i.Scopes,
|
||||
); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -164,7 +172,7 @@ SET revoked_at = NOW()
|
||||
WHERE project_id = $1
|
||||
AND id = $2
|
||||
AND revoked_at IS NULL
|
||||
RETURNING id, project_id, name, prefix, secret_hash, created_by_user_id, last_used_at, revoked_at, created_at
|
||||
RETURNING id, project_id, name, prefix, secret_hash, created_by_user_id, last_used_at, revoked_at, created_at, scopes
|
||||
`
|
||||
|
||||
type RevokeAPIKeyParams struct {
|
||||
@@ -185,6 +193,7 @@ func (q *Queries) RevokeAPIKey(ctx context.Context, arg RevokeAPIKeyParams) (Cor
|
||||
&i.LastUsedAt,
|
||||
&i.RevokedAt,
|
||||
&i.CreatedAt,
|
||||
&i.Scopes,
|
||||
)
|
||||
return i, err
|
||||
}
|
||||
|
||||
@@ -24,7 +24,7 @@ new_org_member AS (
|
||||
new_project AS (
|
||||
INSERT INTO core.projects (organization_id, slug, name, description)
|
||||
SELECT id, $4, $5, $6 FROM new_org
|
||||
RETURNING id, organization_id, slug, name, description, created_at
|
||||
RETURNING id, organization_id, slug, name, description, created_at, retention_events_days, retention_audit_days, retention_webhook_days
|
||||
),
|
||||
new_project_member AS (
|
||||
INSERT INTO core.project_members (project_id, user_id, role)
|
||||
|
||||
@@ -150,6 +150,7 @@ type CoreApiKey struct {
|
||||
LastUsedAt pgtype.Timestamptz `json:"last_used_at"`
|
||||
RevokedAt pgtype.Timestamptz `json:"revoked_at"`
|
||||
CreatedAt pgtype.Timestamptz `json:"created_at"`
|
||||
Scopes []string `json:"scopes"`
|
||||
}
|
||||
|
||||
type CoreAuditLog struct {
|
||||
@@ -273,12 +274,15 @@ type CoreOrganizationMember struct {
|
||||
}
|
||||
|
||||
type CoreProject struct {
|
||||
ID uuid.UUID `json:"id"`
|
||||
OrganizationID uuid.UUID `json:"organization_id"`
|
||||
Slug string `json:"slug"`
|
||||
Name string `json:"name"`
|
||||
Description *string `json:"description"`
|
||||
CreatedAt pgtype.Timestamptz `json:"created_at"`
|
||||
ID uuid.UUID `json:"id"`
|
||||
OrganizationID uuid.UUID `json:"organization_id"`
|
||||
Slug string `json:"slug"`
|
||||
Name string `json:"name"`
|
||||
Description *string `json:"description"`
|
||||
CreatedAt pgtype.Timestamptz `json:"created_at"`
|
||||
RetentionEventsDays int32 `json:"retention_events_days"`
|
||||
RetentionAuditDays int32 `json:"retention_audit_days"`
|
||||
RetentionWebhookDays int32 `json:"retention_webhook_days"`
|
||||
}
|
||||
|
||||
type CoreProjectInvitation struct {
|
||||
|
||||
@@ -59,7 +59,7 @@ INSERT INTO core.projects (
|
||||
name,
|
||||
description
|
||||
) VALUES ($1, $2, $3, $4)
|
||||
RETURNING id, organization_id, slug, name, description, created_at
|
||||
RETURNING id, organization_id, slug, name, description, created_at, retention_events_days, retention_audit_days, retention_webhook_days
|
||||
`
|
||||
|
||||
type CreateProjectParams struct {
|
||||
@@ -84,6 +84,9 @@ func (q *Queries) CreateProject(ctx context.Context, arg CreateProjectParams) (C
|
||||
&i.Name,
|
||||
&i.Description,
|
||||
&i.CreatedAt,
|
||||
&i.RetentionEventsDays,
|
||||
&i.RetentionAuditDays,
|
||||
&i.RetentionWebhookDays,
|
||||
)
|
||||
return i, err
|
||||
}
|
||||
@@ -91,7 +94,7 @@ func (q *Queries) CreateProject(ctx context.Context, arg CreateProjectParams) (C
|
||||
const deleteProjectByID = `-- name: DeleteProjectByID :one
|
||||
DELETE FROM core.projects
|
||||
WHERE id = $1
|
||||
RETURNING id, organization_id, slug, name, description, created_at
|
||||
RETURNING id, organization_id, slug, name, description, created_at, retention_events_days, retention_audit_days, retention_webhook_days
|
||||
`
|
||||
|
||||
func (q *Queries) DeleteProjectByID(ctx context.Context, id uuid.UUID) (CoreProject, error) {
|
||||
@@ -104,12 +107,15 @@ func (q *Queries) DeleteProjectByID(ctx context.Context, id uuid.UUID) (CoreProj
|
||||
&i.Name,
|
||||
&i.Description,
|
||||
&i.CreatedAt,
|
||||
&i.RetentionEventsDays,
|
||||
&i.RetentionAuditDays,
|
||||
&i.RetentionWebhookDays,
|
||||
)
|
||||
return i, err
|
||||
}
|
||||
|
||||
const getProjectByID = `-- name: GetProjectByID :one
|
||||
SELECT id, organization_id, slug, name, description, created_at FROM core.projects
|
||||
SELECT id, organization_id, slug, name, description, created_at, retention_events_days, retention_audit_days, retention_webhook_days FROM core.projects
|
||||
WHERE id = $1
|
||||
`
|
||||
|
||||
@@ -123,6 +129,9 @@ func (q *Queries) GetProjectByID(ctx context.Context, id uuid.UUID) (CoreProject
|
||||
&i.Name,
|
||||
&i.Description,
|
||||
&i.CreatedAt,
|
||||
&i.RetentionEventsDays,
|
||||
&i.RetentionAuditDays,
|
||||
&i.RetentionWebhookDays,
|
||||
)
|
||||
return i, err
|
||||
}
|
||||
@@ -327,7 +336,7 @@ func (q *Queries) ListProjectMembers(ctx context.Context, projectID uuid.UUID) (
|
||||
|
||||
const listProjectsForOrganization = `-- name: ListProjectsForOrganization :many
|
||||
SELECT
|
||||
p.id, p.organization_id, p.slug, p.name, p.description, p.created_at,
|
||||
p.id, p.organization_id, p.slug, p.name, p.description, p.created_at, p.retention_events_days, p.retention_audit_days, p.retention_webhook_days,
|
||||
pm.role AS membership_role
|
||||
FROM core.projects p
|
||||
LEFT JOIN core.project_members pm
|
||||
@@ -350,13 +359,16 @@ type ListProjectsForOrganizationParams struct {
|
||||
}
|
||||
|
||||
type ListProjectsForOrganizationRow struct {
|
||||
ID uuid.UUID `json:"id"`
|
||||
OrganizationID uuid.UUID `json:"organization_id"`
|
||||
Slug string `json:"slug"`
|
||||
Name string `json:"name"`
|
||||
Description *string `json:"description"`
|
||||
CreatedAt pgtype.Timestamptz `json:"created_at"`
|
||||
MembershipRole NullCoreProjectRole `json:"membership_role"`
|
||||
ID uuid.UUID `json:"id"`
|
||||
OrganizationID uuid.UUID `json:"organization_id"`
|
||||
Slug string `json:"slug"`
|
||||
Name string `json:"name"`
|
||||
Description *string `json:"description"`
|
||||
CreatedAt pgtype.Timestamptz `json:"created_at"`
|
||||
RetentionEventsDays int32 `json:"retention_events_days"`
|
||||
RetentionAuditDays int32 `json:"retention_audit_days"`
|
||||
RetentionWebhookDays int32 `json:"retention_webhook_days"`
|
||||
MembershipRole NullCoreProjectRole `json:"membership_role"`
|
||||
}
|
||||
|
||||
func (q *Queries) ListProjectsForOrganization(ctx context.Context, arg ListProjectsForOrganizationParams) ([]ListProjectsForOrganizationRow, error) {
|
||||
@@ -375,6 +387,9 @@ func (q *Queries) ListProjectsForOrganization(ctx context.Context, arg ListProje
|
||||
&i.Name,
|
||||
&i.Description,
|
||||
&i.CreatedAt,
|
||||
&i.RetentionEventsDays,
|
||||
&i.RetentionAuditDays,
|
||||
&i.RetentionWebhookDays,
|
||||
&i.MembershipRole,
|
||||
); err != nil {
|
||||
return nil, err
|
||||
@@ -416,16 +431,22 @@ const updateProjectByID = `-- name: UpdateProjectByID :one
|
||||
UPDATE core.projects
|
||||
SET slug = $2,
|
||||
name = $3,
|
||||
description = $4
|
||||
description = $4,
|
||||
retention_events_days = COALESCE($5, retention_events_days),
|
||||
retention_audit_days = COALESCE($6, retention_audit_days),
|
||||
retention_webhook_days = COALESCE($7, retention_webhook_days)
|
||||
WHERE id = $1
|
||||
RETURNING id, organization_id, slug, name, description, created_at
|
||||
RETURNING id, organization_id, slug, name, description, created_at, retention_events_days, retention_audit_days, retention_webhook_days
|
||||
`
|
||||
|
||||
type UpdateProjectByIDParams struct {
|
||||
ID uuid.UUID `json:"id"`
|
||||
Slug string `json:"slug"`
|
||||
Name string `json:"name"`
|
||||
Description *string `json:"description"`
|
||||
ID uuid.UUID `json:"id"`
|
||||
Slug string `json:"slug"`
|
||||
Name string `json:"name"`
|
||||
Description *string `json:"description"`
|
||||
RetentionEventsDays *int32 `json:"retention_events_days"`
|
||||
RetentionAuditDays *int32 `json:"retention_audit_days"`
|
||||
RetentionWebhookDays *int32 `json:"retention_webhook_days"`
|
||||
}
|
||||
|
||||
func (q *Queries) UpdateProjectByID(ctx context.Context, arg UpdateProjectByIDParams) (CoreProject, error) {
|
||||
@@ -434,6 +455,9 @@ func (q *Queries) UpdateProjectByID(ctx context.Context, arg UpdateProjectByIDPa
|
||||
arg.Slug,
|
||||
arg.Name,
|
||||
arg.Description,
|
||||
arg.RetentionEventsDays,
|
||||
arg.RetentionAuditDays,
|
||||
arg.RetentionWebhookDays,
|
||||
)
|
||||
var i CoreProject
|
||||
err := row.Scan(
|
||||
@@ -443,6 +467,9 @@ func (q *Queries) UpdateProjectByID(ctx context.Context, arg UpdateProjectByIDPa
|
||||
&i.Name,
|
||||
&i.Description,
|
||||
&i.CreatedAt,
|
||||
&i.RetentionEventsDays,
|
||||
&i.RetentionAuditDays,
|
||||
&i.RetentionWebhookDays,
|
||||
)
|
||||
return i, err
|
||||
}
|
||||
|
||||
@@ -100,6 +100,15 @@ type Querier interface {
|
||||
RemoveProjectMember(ctx context.Context, arg RemoveProjectMemberParams) (CoreProjectMember, error)
|
||||
RemoveProjectMembershipsForOrganizationUser(ctx context.Context, arg RemoveProjectMembershipsForOrganizationUserParams) error
|
||||
RevokeAPIKey(ctx context.Context, arg RevokeAPIKeyParams) (CoreApiKey, error)
|
||||
SweepExpiredAuditLogs(ctx context.Context) (int64, error)
|
||||
// Per-project retention sweeps. A retention of 0 days disables the sweep for
|
||||
// that stream. Pending webhook deliveries are never swept — they may still be
|
||||
// retrying.
|
||||
SweepExpiredEvents(ctx context.Context) (int64, error)
|
||||
// Org-scoped audit rows (project_id NULL) have no project retention — sweep
|
||||
// them on a fixed 365-day floor so they cannot grow unbounded.
|
||||
SweepExpiredOrgAuditLogs(ctx context.Context) (int64, error)
|
||||
SweepExpiredWebhookDeliveries(ctx context.Context) (int64, error)
|
||||
TouchAPIKey(ctx context.Context, id uuid.UUID) error
|
||||
UpdateBucketByID(ctx context.Context, arg UpdateBucketByIDParams) (CoreBucket, error)
|
||||
UpdateCollection(ctx context.Context, arg UpdateCollectionParams) (CoreCollection, error)
|
||||
|
||||
@@ -0,0 +1,80 @@
|
||||
// Code generated by sqlc. DO NOT EDIT.
|
||||
// versions:
|
||||
// sqlc v1.29.0
|
||||
// source: retention.sql
|
||||
|
||||
package db
|
||||
|
||||
import (
|
||||
"context"
|
||||
)
|
||||
|
||||
const sweepExpiredAuditLogs = `-- name: SweepExpiredAuditLogs :execrows
|
||||
DELETE FROM core.audit_logs a
|
||||
USING core.projects p
|
||||
WHERE a.project_id = p.id
|
||||
AND p.retention_audit_days > 0
|
||||
AND a.created_at < NOW() - make_interval(days => p.retention_audit_days)
|
||||
`
|
||||
|
||||
func (q *Queries) SweepExpiredAuditLogs(ctx context.Context) (int64, error) {
|
||||
result, err := q.db.Exec(ctx, sweepExpiredAuditLogs)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return result.RowsAffected(), nil
|
||||
}
|
||||
|
||||
const sweepExpiredEvents = `-- name: SweepExpiredEvents :execrows
|
||||
|
||||
DELETE FROM core.events e
|
||||
USING core.projects p
|
||||
WHERE e.project_id = p.id
|
||||
AND p.retention_events_days > 0
|
||||
AND e.ts < NOW() - make_interval(days => p.retention_events_days)
|
||||
`
|
||||
|
||||
// Per-project retention sweeps. A retention of 0 days disables the sweep for
|
||||
// that stream. Pending webhook deliveries are never swept — they may still be
|
||||
// retrying.
|
||||
func (q *Queries) SweepExpiredEvents(ctx context.Context) (int64, error) {
|
||||
result, err := q.db.Exec(ctx, sweepExpiredEvents)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return result.RowsAffected(), nil
|
||||
}
|
||||
|
||||
const sweepExpiredOrgAuditLogs = `-- name: SweepExpiredOrgAuditLogs :execrows
|
||||
DELETE FROM core.audit_logs a
|
||||
WHERE a.project_id IS NULL
|
||||
AND a.created_at < NOW() - INTERVAL '365 days'
|
||||
`
|
||||
|
||||
// Org-scoped audit rows (project_id NULL) have no project retention — sweep
|
||||
// them on a fixed 365-day floor so they cannot grow unbounded.
|
||||
func (q *Queries) SweepExpiredOrgAuditLogs(ctx context.Context) (int64, error) {
|
||||
result, err := q.db.Exec(ctx, sweepExpiredOrgAuditLogs)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return result.RowsAffected(), nil
|
||||
}
|
||||
|
||||
const sweepExpiredWebhookDeliveries = `-- name: SweepExpiredWebhookDeliveries :execrows
|
||||
DELETE FROM core.webhook_deliveries d
|
||||
USING core.webhooks w
|
||||
JOIN core.projects p ON p.id = w.project_id
|
||||
WHERE d.webhook_id = w.id
|
||||
AND p.retention_webhook_days > 0
|
||||
AND d.status <> 'pending'
|
||||
AND d.created_at < NOW() - make_interval(days => p.retention_webhook_days)
|
||||
`
|
||||
|
||||
func (q *Queries) SweepExpiredWebhookDeliveries(ctx context.Context) (int64, error) {
|
||||
result, err := q.db.Exec(ctx, sweepExpiredWebhookDeliveries)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return result.RowsAffected(), nil
|
||||
}
|
||||
@@ -239,6 +239,7 @@ const listComponents = `-- name: ListComponents :many
|
||||
SELECT id, project_id, name, kind, meta, created_at FROM core.components
|
||||
WHERE project_id = $1
|
||||
ORDER BY name
|
||||
LIMIT 500
|
||||
`
|
||||
|
||||
func (q *Queries) ListComponents(ctx context.Context, projectID uuid.UUID) ([]CoreComponent, error) {
|
||||
|
||||
@@ -204,6 +204,23 @@ func (h *HTTPHandler) dbxRedis(c *gin.Context) {
|
||||
c.JSON(http.StatusOK, result)
|
||||
}
|
||||
|
||||
func (h *HTTPHandler) dbxTransfer(c *gin.Context) {
|
||||
actor, projectID, connectionID, ok := h.actorProjectConn(c)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
var body services.TransferInput
|
||||
if !h.bindAndValidate(c, &body) {
|
||||
return
|
||||
}
|
||||
result, err := h.Platform.TransferDBRows(c.Request.Context(), actor, projectID, connectionID, body, middleware.RequestIDFromContext(c))
|
||||
if err != nil {
|
||||
h.dbxError(c, err)
|
||||
return
|
||||
}
|
||||
c.JSON(http.StatusOK, result)
|
||||
}
|
||||
|
||||
// actorAndProject resolves the actor and :projectID param.
|
||||
func (h *HTTPHandler) actorAndProject(c *gin.Context) (*models.Actor, uuid.UUID, bool) {
|
||||
actor, ok := middleware.RequireActor(c)
|
||||
|
||||
@@ -29,6 +29,7 @@ func (h *HTTPHandler) Register(router *gin.Engine) {
|
||||
router.GET("/api/v1/health/liveness", h.liveness)
|
||||
router.GET("/api/v1/health/readiness", h.readiness)
|
||||
router.GET("/api/v1/openapi.yaml", h.openapi)
|
||||
router.GET("/api/v1/metrics", h.metrics)
|
||||
|
||||
api := router.Group("/api/v1")
|
||||
api.GET("/me", h.me)
|
||||
@@ -91,6 +92,7 @@ func (h *HTTPHandler) Register(router *gin.Engine) {
|
||||
api.GET("/projects/:projectID/db-connections/:connectionID/foreign-keys", h.dbxForeignKeys)
|
||||
api.POST("/projects/:projectID/db-connections/:connectionID/query", h.dbxQuery)
|
||||
api.POST("/projects/:projectID/db-connections/:connectionID/redis", h.dbxRedis)
|
||||
api.POST("/projects/:projectID/db-connections/:connectionID/transfer", h.dbxTransfer)
|
||||
api.GET("/projects/:projectID/integrations", h.listIntegrations)
|
||||
api.POST("/projects/:projectID/integrations", h.createIntegration)
|
||||
api.DELETE("/projects/:projectID/integrations/:integrationID", h.deleteIntegration)
|
||||
@@ -123,6 +125,12 @@ func (h *HTTPHandler) openapi(c *gin.Context) {
|
||||
c.File("openapi/openapi.yaml")
|
||||
}
|
||||
|
||||
// metrics is intentionally unauthenticated — Prometheus scrapes it. Restrict
|
||||
// it at the edge if the deployment is public-facing.
|
||||
func (h *HTTPHandler) metrics(c *gin.Context) {
|
||||
c.Data(http.StatusOK, "text/plain; version=0.0.4; charset=utf-8", []byte(h.Metrics.RenderPrometheus()))
|
||||
}
|
||||
|
||||
func (h *HTTPHandler) me(c *gin.Context) {
|
||||
actor, ok := middleware.RequireActor(c)
|
||||
if !ok {
|
||||
|
||||
@@ -17,12 +17,15 @@ type organizationMembershipResponse struct {
|
||||
}
|
||||
|
||||
type projectResponse struct {
|
||||
ID string `json:"id"`
|
||||
OrganizationID string `json:"organization_id"`
|
||||
Slug string `json:"slug"`
|
||||
Name string `json:"name"`
|
||||
Description *string `json:"description"`
|
||||
MembershipRole *string `json:"membership_role"`
|
||||
ID string `json:"id"`
|
||||
OrganizationID string `json:"organization_id"`
|
||||
Slug string `json:"slug"`
|
||||
Name string `json:"name"`
|
||||
Description *string `json:"description"`
|
||||
MembershipRole *string `json:"membership_role"`
|
||||
RetentionEventsDays int32 `json:"retention_events_days"`
|
||||
RetentionAuditDays int32 `json:"retention_audit_days"`
|
||||
RetentionWebhookDays int32 `json:"retention_webhook_days"`
|
||||
}
|
||||
|
||||
type apiKeyResponse struct {
|
||||
@@ -30,6 +33,7 @@ type apiKeyResponse struct {
|
||||
ProjectID string `json:"project_id"`
|
||||
Name string `json:"name"`
|
||||
Prefix string `json:"prefix"`
|
||||
Scopes []string `json:"scopes"`
|
||||
LastUsedAt *time.Time `json:"last_used_at"`
|
||||
RevokedAt *time.Time `json:"revoked_at"`
|
||||
}
|
||||
@@ -114,23 +118,29 @@ func toProjectFromRow(row db.ListProjectsForOrganizationRow) projectResponse {
|
||||
membershipRole = &role
|
||||
}
|
||||
return projectResponse{
|
||||
ID: row.ID.String(),
|
||||
OrganizationID: row.OrganizationID.String(),
|
||||
Slug: row.Slug,
|
||||
Name: row.Name,
|
||||
Description: row.Description,
|
||||
MembershipRole: membershipRole,
|
||||
ID: row.ID.String(),
|
||||
OrganizationID: row.OrganizationID.String(),
|
||||
Slug: row.Slug,
|
||||
Name: row.Name,
|
||||
Description: row.Description,
|
||||
MembershipRole: membershipRole,
|
||||
RetentionEventsDays: row.RetentionEventsDays,
|
||||
RetentionAuditDays: row.RetentionAuditDays,
|
||||
RetentionWebhookDays: row.RetentionWebhookDays,
|
||||
}
|
||||
}
|
||||
|
||||
func toProjectFromCore(row db.CoreProject) projectResponse {
|
||||
return projectResponse{
|
||||
ID: row.ID.String(),
|
||||
OrganizationID: row.OrganizationID.String(),
|
||||
Slug: row.Slug,
|
||||
Name: row.Name,
|
||||
Description: row.Description,
|
||||
MembershipRole: nil,
|
||||
ID: row.ID.String(),
|
||||
OrganizationID: row.OrganizationID.String(),
|
||||
Slug: row.Slug,
|
||||
Name: row.Name,
|
||||
Description: row.Description,
|
||||
MembershipRole: nil,
|
||||
RetentionEventsDays: row.RetentionEventsDays,
|
||||
RetentionAuditDays: row.RetentionAuditDays,
|
||||
RetentionWebhookDays: row.RetentionWebhookDays,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -148,6 +158,7 @@ func toAPIKeyResponse(row db.CoreApiKey) apiKeyResponse {
|
||||
ProjectID: row.ProjectID.String(),
|
||||
Name: row.Name,
|
||||
Prefix: row.Prefix,
|
||||
Scopes: row.Scopes,
|
||||
LastUsedAt: timestamptzPtr(row.LastUsedAt),
|
||||
RevokedAt: timestamptzPtr(row.RevokedAt),
|
||||
}
|
||||
|
||||
@@ -125,6 +125,10 @@ func (m AuthMiddleware) ResolveActor() gin.HandlerFunc {
|
||||
if !m.enforceRateLimit(c, "apikey", actor.APIKeyPrefix, m.RateLimits.APIKeyPerMinute, "API key rate limit exceeded") {
|
||||
return
|
||||
}
|
||||
if need := requiredAPIKeyScope(c.Request.Method, c.Request.URL.Path); !actor.HasScope(need) {
|
||||
apperrors.Abort(c, http.StatusForbidden, "insufficient_scope", "api key lacks the required scope: "+need)
|
||||
return
|
||||
}
|
||||
c.Set(actorKey, actor)
|
||||
}
|
||||
c.Next()
|
||||
@@ -264,9 +268,36 @@ func (m AuthMiddleware) resolveAPIKeyActor(ctx context.Context, rawKey string) (
|
||||
OrganizationID: &orgID,
|
||||
APIKeyID: &apiKeyID,
|
||||
APIKeyPrefix: prefix,
|
||||
Scopes: row.Scopes,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// requiredAPIKeyScope maps a request to the scope an API key must hold.
|
||||
// admin-only surfaces: org management, invitations, bootstrap, project
|
||||
// settings/members, and API key minting (a write key must never mint keys).
|
||||
func requiredAPIKeyScope(method, path string) string {
|
||||
switch method {
|
||||
case http.MethodGet, http.MethodHead, http.MethodOptions:
|
||||
return "read"
|
||||
}
|
||||
if path == "/api/v1/ingest" {
|
||||
return "ingest"
|
||||
}
|
||||
rest := strings.TrimPrefix(path, "/api/v1/")
|
||||
segs := strings.SplitN(rest, "/", 3)
|
||||
switch segs[0] {
|
||||
case "organizations", "invitations", "bootstrap":
|
||||
return "admin"
|
||||
case "projects":
|
||||
if len(segs) < 3 ||
|
||||
strings.HasPrefix(segs[2], "api-keys") ||
|
||||
strings.HasPrefix(segs[2], "members") {
|
||||
return "admin"
|
||||
}
|
||||
}
|
||||
return "write"
|
||||
}
|
||||
|
||||
func PgUUID(id uuid.UUID) pgtype.UUID {
|
||||
return pgtype.UUID{Bytes: id, Valid: true}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,66 @@
|
||||
package middleware
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"testing"
|
||||
|
||||
"github.com/tdvorak/primora/apps/backend/internal/models"
|
||||
)
|
||||
|
||||
func TestRequiredAPIKeyScope(t *testing.T) {
|
||||
cases := []struct {
|
||||
method, path, want string
|
||||
}{
|
||||
{http.MethodPost, "/api/v1/ingest", "ingest"},
|
||||
{http.MethodOptions, "/api/v1/ingest", "read"}, // preflight passes through CORS
|
||||
{http.MethodGet, "/api/v1/projects/x/events", "read"},
|
||||
{http.MethodGet, "/api/v1/projects/x/api-keys", "read"},
|
||||
{http.MethodPost, "/api/v1/projects/x/api-keys", "admin"},
|
||||
{http.MethodDelete, "/api/v1/projects/x/api-keys/y", "admin"},
|
||||
{http.MethodPatch, "/api/v1/projects/x/members/y", "admin"},
|
||||
{http.MethodPatch, "/api/v1/projects/x", "admin"},
|
||||
{http.MethodDelete, "/api/v1/projects/x", "admin"},
|
||||
{http.MethodPatch, "/api/v1/organizations/x", "admin"},
|
||||
{http.MethodDelete, "/api/v1/organizations/x", "admin"},
|
||||
{http.MethodPost, "/api/v1/organizations/x/invitations", "admin"},
|
||||
{http.MethodPost, "/api/v1/bootstrap", "admin"},
|
||||
{http.MethodPost, "/api/v1/projects/x/buckets", "write"},
|
||||
{http.MethodPost, "/api/v1/projects/x/db-connections/y/transfer", "write"},
|
||||
{http.MethodDelete, "/api/v1/buckets/x", "write"},
|
||||
{http.MethodPost, "/api/v1/projects/x/deploy-markers", "write"},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
if got := requiredAPIKeyScope(tc.method, tc.path); got != tc.want {
|
||||
t.Errorf("%s %s: got %q, want %q", tc.method, tc.path, got, tc.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestActorHasScope(t *testing.T) {
|
||||
key := func(scopes ...string) *models.Actor {
|
||||
return &models.Actor{Type: models.ActorTypeAPIKey, Scopes: scopes}
|
||||
}
|
||||
user := &models.Actor{Type: models.ActorTypeUser}
|
||||
|
||||
if !user.HasScope("admin") {
|
||||
t.Error("user actors bypass scope checks")
|
||||
}
|
||||
if !key("admin").HasScope("read") {
|
||||
t.Error("admin should satisfy every requirement")
|
||||
}
|
||||
if !key("write").HasScope("read") || !key("write").HasScope("ingest") {
|
||||
t.Error("write should imply read and ingest")
|
||||
}
|
||||
if key("write").HasScope("admin") {
|
||||
t.Error("write must not imply admin")
|
||||
}
|
||||
if key("read").HasScope("write") || key("read").HasScope("ingest") {
|
||||
t.Error("read must not imply write or ingest")
|
||||
}
|
||||
if !key("ingest").HasScope("ingest") || key("ingest").HasScope("read") {
|
||||
t.Error("ingest keys only reach /ingest")
|
||||
}
|
||||
if key().HasScope("read") {
|
||||
t.Error("scopeless key has no access")
|
||||
}
|
||||
}
|
||||
@@ -21,6 +21,7 @@ type Actor struct {
|
||||
OrganizationID *uuid.UUID
|
||||
APIKeyID *uuid.UUID
|
||||
APIKeyPrefix string
|
||||
Scopes []string
|
||||
}
|
||||
|
||||
func (a *Actor) IsUser() bool {
|
||||
@@ -30,3 +31,35 @@ func (a *Actor) IsUser() bool {
|
||||
func (a *Actor) IsAPIKey() bool {
|
||||
return a != nil && a.Type == ActorTypeAPIKey
|
||||
}
|
||||
|
||||
// HasScope reports whether the actor may perform work requiring `need`.
|
||||
// User actors are always allowed — scopes constrain API keys only.
|
||||
// Scope semantics: admin unlocks everything; write unlocks read + ingest +
|
||||
// non-admin mutations; read and ingest unlock only themselves.
|
||||
func (a *Actor) HasScope(need string) bool {
|
||||
if a == nil {
|
||||
return false
|
||||
}
|
||||
if a.IsUser() {
|
||||
return true
|
||||
}
|
||||
for _, s := range a.Scopes {
|
||||
switch s {
|
||||
case "admin":
|
||||
return true
|
||||
case "write":
|
||||
if need == "read" || need == "write" || need == "ingest" {
|
||||
return true
|
||||
}
|
||||
case "read":
|
||||
if need == "read" {
|
||||
return true
|
||||
}
|
||||
case "ingest":
|
||||
if need == "ingest" {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
@@ -1,6 +1,7 @@
|
||||
package observability
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
@@ -43,12 +44,12 @@ func (m *Metrics) DecrementActive() {
|
||||
func (m *Metrics) GetStats() map[string]any {
|
||||
m.mu.RLock()
|
||||
defer m.mu.RUnlock()
|
||||
|
||||
|
||||
avgResponseTime := int64(0)
|
||||
if m.requestCount > 0 {
|
||||
avgResponseTime = m.totalResponseTime.Milliseconds() / m.requestCount
|
||||
}
|
||||
|
||||
|
||||
return map[string]any{
|
||||
"total_requests": m.requestCount,
|
||||
"total_errors": m.errorCount,
|
||||
@@ -56,3 +57,24 @@ func (m *Metrics) GetStats() map[string]any {
|
||||
"avg_response_time_ms": avgResponseTime,
|
||||
}
|
||||
}
|
||||
|
||||
// RenderPrometheus emits the collector in Prometheus text exposition format.
|
||||
// Duration is exposed as sum+count so scrapers can compute real averages.
|
||||
func (m *Metrics) RenderPrometheus() string {
|
||||
m.mu.RLock()
|
||||
defer m.mu.RUnlock()
|
||||
return fmt.Sprintf(`# HELP primora_http_requests_total Total HTTP requests handled.
|
||||
# TYPE primora_http_requests_total counter
|
||||
primora_http_requests_total %d
|
||||
# HELP primora_http_request_errors_total HTTP requests that ended in a 5xx response.
|
||||
# TYPE primora_http_request_errors_total counter
|
||||
primora_http_request_errors_total %d
|
||||
# HELP primora_http_active_requests Requests currently in flight.
|
||||
# TYPE primora_http_active_requests gauge
|
||||
primora_http_active_requests %d
|
||||
# HELP primora_http_request_duration_ms_sum Sum of request durations in milliseconds.
|
||||
# TYPE primora_http_request_duration_ms_sum counter
|
||||
primora_http_request_duration_ms_sum %d
|
||||
`,
|
||||
m.requestCount, m.errorCount, m.activeRequests, m.totalResponseTime.Milliseconds())
|
||||
}
|
||||
@@ -500,3 +500,223 @@ func (s *PlatformService) ExecuteDBXRedis(ctx context.Context, actor *models.Act
|
||||
}
|
||||
return map[string]any{"output": text}, nil
|
||||
}
|
||||
|
||||
// TransferInput describes a DB-to-DB link: rows read from the source
|
||||
// connection are written to the target connection.
|
||||
type TransferInput struct {
|
||||
TargetConnectionID string `json:"target_connection_id" validate:"required,uuid"`
|
||||
Query string `json:"query" validate:"required"`
|
||||
Database string `json:"database"`
|
||||
// SQL targets: destination table (schema-qualified allowed).
|
||||
TargetTable string `json:"target_table"`
|
||||
// Redis targets: SET key template with {column} placeholders; the value is
|
||||
// value_column's cell, or the whole row as JSON when omitted.
|
||||
KeyPattern string `json:"key_pattern"`
|
||||
ValueColumn string `json:"value_column"`
|
||||
// Optional TTL in seconds applied to each Redis SET.
|
||||
TTLSeconds *int `json:"ttl_seconds" validate:"omitempty,min=0"`
|
||||
Limit int `json:"limit" validate:"omitempty,min=1,max=1000"`
|
||||
}
|
||||
|
||||
const transferMaxRows = 1000
|
||||
|
||||
// TransferDBRows moves data between two saved connections of the same
|
||||
// project. Both connections must accept write-level access.
|
||||
func (s *PlatformService) TransferDBRows(ctx context.Context, actor *models.Actor, projectID, sourceID uuid.UUID, input TransferInput, requestID string) (map[string]any, error) {
|
||||
source, err := s.connForTool(ctx, actor, projectID, sourceID, true)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
targetID, err := uuid.Parse(input.TargetConnectionID)
|
||||
if err != nil {
|
||||
return nil, errors.New("target_connection_id must be a uuid")
|
||||
}
|
||||
target, err := s.connForTool(ctx, actor, projectID, targetID, true)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
limit := input.Limit
|
||||
if limit <= 0 || limit > transferMaxRows {
|
||||
limit = transferMaxRows
|
||||
}
|
||||
args := map[string]any{"connection_name": dbxConnectionName(projectID, source.Name), "sql": input.Query}
|
||||
if input.Database != "" {
|
||||
args["database"] = input.Database
|
||||
}
|
||||
text, err := s.dbx.Call(ctx, "dbx_execute_query", args)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
table, _ := dbx.ParseMDTable(text)
|
||||
if table == nil || len(table.Columns) == 0 {
|
||||
return nil, errors.New("source query did not return a result table")
|
||||
}
|
||||
rows := table.Rows
|
||||
truncated := false
|
||||
if len(rows) > limit {
|
||||
rows = rows[:limit]
|
||||
truncated = true
|
||||
}
|
||||
|
||||
var transferred int
|
||||
if target.DbType == "redis" {
|
||||
transferred, err = s.transferToRedis(ctx, projectID, target, table.Columns, rows, input)
|
||||
} else {
|
||||
transferred, err = s.transferToSQL(ctx, projectID, target, table.Columns, rows, input)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
project, perr := s.repo.Queries().GetProjectByID(ctx, projectID)
|
||||
if perr == nil {
|
||||
_, _ = s.repo.Queries().CreateAuditLog(ctx, newAuditParams(project.OrganizationID, projectID, actor, requestID, "db_connection.transferred", "db_connection", source.ID.String(), map[string]any{
|
||||
"source": source.Name,
|
||||
"target": target.Name,
|
||||
"transferred": transferred,
|
||||
"truncated": truncated,
|
||||
}))
|
||||
}
|
||||
return map[string]any{
|
||||
"transferred": transferred,
|
||||
"truncated": truncated,
|
||||
"source": source.Name,
|
||||
"target": target.Name,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// transferToSQL INSERTs rows into target_table in batches of 100. Cell values
|
||||
// arrive as text from DBX's markdown output; literal "NULL" maps to SQL NULL.
|
||||
func (s *PlatformService) transferToSQL(ctx context.Context, projectID uuid.UUID, target db.CoreDbConnection, cols []string, rows [][]string, input TransferInput) (int, error) {
|
||||
tableIdent, ok := safeDottedIdent(input.TargetTable)
|
||||
if !ok {
|
||||
return 0, errors.New("target_table must be a plain identifier or schema.table")
|
||||
}
|
||||
colIdents := make([]string, len(cols))
|
||||
for i, col := range cols {
|
||||
ident, ok := safeIdent(col)
|
||||
if !ok {
|
||||
return 0, fmt.Errorf("source column %q is not a usable identifier", col)
|
||||
}
|
||||
colIdents[i] = `"` + ident + `"`
|
||||
}
|
||||
|
||||
transferred := 0
|
||||
const batch = 100
|
||||
for start := 0; start < len(rows); start += batch {
|
||||
end := min(start+batch, len(rows))
|
||||
var values strings.Builder
|
||||
values.WriteString("INSERT INTO " + tableIdent + " (" + strings.Join(colIdents, ", ") + ") VALUES ")
|
||||
for i, row := range rows[start:end] {
|
||||
if i > 0 {
|
||||
values.WriteString(", ")
|
||||
}
|
||||
values.WriteString("(")
|
||||
for j := range cols {
|
||||
if j > 0 {
|
||||
values.WriteString(", ")
|
||||
}
|
||||
cell := ""
|
||||
if j < len(row) {
|
||||
cell = row[j]
|
||||
}
|
||||
if cell == "NULL" {
|
||||
values.WriteString("NULL")
|
||||
} else {
|
||||
values.WriteString(sqlQuote(cell))
|
||||
}
|
||||
}
|
||||
values.WriteString(")")
|
||||
}
|
||||
if _, err := s.dbx.Call(ctx, "dbx_execute_query", map[string]any{
|
||||
"connection_name": dbxConnectionName(projectID, target.Name),
|
||||
"sql": values.String(),
|
||||
}); err != nil {
|
||||
return transferred, fmt.Errorf("insert batch at row %d: %w", start, err)
|
||||
}
|
||||
transferred += end - start
|
||||
}
|
||||
return transferred, nil
|
||||
}
|
||||
|
||||
// transferToRedis SETs one key per row. Placeholders like {email} in
|
||||
// key_pattern are replaced with that row's cell value.
|
||||
var keyPlaceholder = regexp.MustCompile(`\{([A-Za-z0-9_]+)\}`)
|
||||
|
||||
func (s *PlatformService) transferToRedis(ctx context.Context, projectID uuid.UUID, target db.CoreDbConnection, cols []string, rows [][]string, input TransferInput) (int, error) {
|
||||
if input.KeyPattern == "" {
|
||||
return 0, errors.New("key_pattern is required for redis targets")
|
||||
}
|
||||
colIndex := map[string]int{}
|
||||
for i, col := range cols {
|
||||
colIndex[col] = i
|
||||
}
|
||||
for _, ph := range keyPlaceholder.FindAllStringSubmatch(input.KeyPattern, -1) {
|
||||
if _, ok := colIndex[ph[1]]; !ok {
|
||||
return 0, fmt.Errorf("key_pattern references unknown column %q", ph[1])
|
||||
}
|
||||
}
|
||||
if input.ValueColumn != "" {
|
||||
if _, ok := colIndex[input.ValueColumn]; !ok {
|
||||
return 0, fmt.Errorf("value_column %q not in source columns", input.ValueColumn)
|
||||
}
|
||||
}
|
||||
|
||||
transferred := 0
|
||||
for _, row := range rows {
|
||||
key := keyPlaceholder.ReplaceAllStringFunc(input.KeyPattern, func(m string) string {
|
||||
col := keyPlaceholder.FindStringSubmatch(m)[1]
|
||||
if i, ok := colIndex[col]; ok && i < len(row) {
|
||||
return row[i]
|
||||
}
|
||||
return ""
|
||||
})
|
||||
var value string
|
||||
if input.ValueColumn != "" {
|
||||
value = row[colIndex[input.ValueColumn]]
|
||||
} else {
|
||||
obj := map[string]string{}
|
||||
for i, col := range cols {
|
||||
if i < len(row) {
|
||||
obj[col] = row[i]
|
||||
}
|
||||
}
|
||||
raw, _ := json.Marshal(obj)
|
||||
value = string(raw)
|
||||
}
|
||||
cmd := "SET " + redisQuote(key) + " " + redisQuote(value)
|
||||
if input.TTLSeconds != nil && *input.TTLSeconds > 0 {
|
||||
cmd += fmt.Sprintf(" EX %d", *input.TTLSeconds)
|
||||
}
|
||||
if _, err := s.dbx.Call(ctx, "dbx_execute_redis_command", map[string]any{
|
||||
"connection_name": dbxConnectionName(projectID, target.Name),
|
||||
"command": cmd,
|
||||
}); err != nil {
|
||||
return transferred, fmt.Errorf("SET %q: %w", key, err)
|
||||
}
|
||||
transferred++
|
||||
}
|
||||
return transferred, nil
|
||||
}
|
||||
|
||||
var dottedIdent = regexp.MustCompile(`^[A-Za-z0-9_]+(\.[A-Za-z0-9_]+)?$`)
|
||||
|
||||
func safeDottedIdent(s string) (string, bool) {
|
||||
if !dottedIdent.MatchString(s) {
|
||||
return "", false
|
||||
}
|
||||
parts := strings.Split(s, ".")
|
||||
for i := range parts {
|
||||
parts[i] = `"` + parts[i] + `"`
|
||||
}
|
||||
return strings.Join(parts, "."), true
|
||||
}
|
||||
|
||||
func sqlQuote(v string) string {
|
||||
return "'" + strings.ReplaceAll(v, "'", "''") + "'"
|
||||
}
|
||||
|
||||
func redisQuote(v string) string {
|
||||
return `"` + strings.ReplaceAll(strings.ReplaceAll(v, `\`, `\\`), `"`, `\"`) + `"`
|
||||
}
|
||||
@@ -67,6 +67,11 @@ type UpdateProjectInput struct {
|
||||
Name string `json:"name" validate:"required,min=2"`
|
||||
Slug string `json:"slug" validate:"required,min=2"`
|
||||
Description *string `json:"description"`
|
||||
// Retention windows in days; nil leaves the current value, 0 disables
|
||||
// the sweeper for that stream.
|
||||
RetentionEventsDays *int32 `json:"retention_events_days" validate:"omitempty,min=0,max=3650"`
|
||||
RetentionAuditDays *int32 `json:"retention_audit_days" validate:"omitempty,min=0,max=3650"`
|
||||
RetentionWebhookDays *int32 `json:"retention_webhook_days" validate:"omitempty,min=0,max=3650"`
|
||||
}
|
||||
|
||||
type CreateInvitationInput struct {
|
||||
@@ -78,7 +83,8 @@ type CreateInvitationInput struct {
|
||||
}
|
||||
|
||||
type CreateAPIKeyInput struct {
|
||||
Name string `json:"name" validate:"required,min=2"`
|
||||
Name string `json:"name" validate:"required,min=2"`
|
||||
Scopes []string `json:"scopes" validate:"omitempty,dive,oneof=ingest read write admin"`
|
||||
}
|
||||
|
||||
type CreateBucketInput struct {
|
||||
@@ -736,10 +742,13 @@ func (s *PlatformService) UpdateProject(ctx context.Context, actor *models.Actor
|
||||
return db.CoreProject{}, err
|
||||
}
|
||||
updated, err := s.repo.Queries().UpdateProjectByID(ctx, db.UpdateProjectByIDParams{
|
||||
ID: projectID,
|
||||
Slug: normalizeSlug(input.Slug),
|
||||
Name: strings.TrimSpace(input.Name),
|
||||
Description: input.Description,
|
||||
ID: projectID,
|
||||
Slug: normalizeSlug(input.Slug),
|
||||
Name: strings.TrimSpace(input.Name),
|
||||
Description: input.Description,
|
||||
RetentionEventsDays: input.RetentionEventsDays,
|
||||
RetentionAuditDays: input.RetentionAuditDays,
|
||||
RetentionWebhookDays: input.RetentionWebhookDays,
|
||||
})
|
||||
if err != nil {
|
||||
return db.CoreProject{}, err
|
||||
@@ -1038,6 +1047,7 @@ func (s *PlatformService) CreateAPIKey(ctx context.Context, actor *models.Actor,
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("generate api key secret: %w", err)
|
||||
}
|
||||
scopes := normalizeAPIKeyScopes(input.Scopes)
|
||||
prefix := "prm_" + prefixSuffix
|
||||
rawKey := prefix + "_" + keySuffix
|
||||
secretHash := sha256.Sum256([]byte(rawKey))
|
||||
@@ -1046,6 +1056,7 @@ func (s *PlatformService) CreateAPIKey(ctx context.Context, actor *models.Actor,
|
||||
Name: strings.TrimSpace(input.Name),
|
||||
Prefix: prefix,
|
||||
SecretHash: secretHash[:],
|
||||
Scopes: scopes,
|
||||
CreatedByUserID: pgtype.UUID{Bytes: *actor.UserID, Valid: true},
|
||||
})
|
||||
if err != nil {
|
||||
@@ -1062,9 +1073,27 @@ func (s *PlatformService) CreateAPIKey(ctx context.Context, actor *models.Actor,
|
||||
"prefix": row.Prefix,
|
||||
"secret": rawKey,
|
||||
"name": row.Name,
|
||||
"scopes": row.Scopes,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// normalizeAPIKeyScopes dedupes the requested scopes. An omitted or empty
|
||||
// list means a full-access key — the pre-scopes behaviour.
|
||||
func normalizeAPIKeyScopes(input []string) []string {
|
||||
if len(input) == 0 {
|
||||
return []string{"admin"}
|
||||
}
|
||||
seen := map[string]bool{}
|
||||
out := make([]string, 0, len(input))
|
||||
for _, s := range input {
|
||||
if !seen[s] {
|
||||
seen[s] = true
|
||||
out = append(out, s)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func (s *PlatformService) RevokeAPIKey(ctx context.Context, actor *models.Actor, projectID, apiKeyID uuid.UUID, requestID string) error {
|
||||
if err := s.requireProjectRole(ctx, actor, projectID, "admin"); err != nil {
|
||||
return err
|
||||
|
||||
@@ -0,0 +1,63 @@
|
||||
package services
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log/slog"
|
||||
"time"
|
||||
|
||||
"github.com/tdvorak/primora/apps/backend/internal/repositories"
|
||||
)
|
||||
|
||||
// RetentionSweeper deletes telemetry events, audit rows, and webhook
|
||||
// deliveries that are past each project's retention window. It runs once
|
||||
// shortly after boot and then hourly — a missed sweep is not a problem, the
|
||||
// next tick catches it.
|
||||
type RetentionSweeper struct {
|
||||
repo *repositories.CoreRepository
|
||||
logger *slog.Logger
|
||||
interval time.Duration
|
||||
}
|
||||
|
||||
func NewRetentionSweeper(repo *repositories.CoreRepository, logger *slog.Logger) *RetentionSweeper {
|
||||
return &RetentionSweeper{repo: repo, logger: logger, interval: time.Hour}
|
||||
}
|
||||
|
||||
func (s *RetentionSweeper) Run(ctx context.Context) {
|
||||
// jarvis: initial delay — let migrations and first-party seeds settle before sweeping.
|
||||
timer := time.NewTimer(time.Minute)
|
||||
defer timer.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-timer.C:
|
||||
s.Sweep(ctx)
|
||||
timer.Reset(s.interval)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (s *RetentionSweeper) Sweep(ctx context.Context) {
|
||||
q := s.repo.Queries()
|
||||
for _, sweep := range []struct {
|
||||
name string
|
||||
fn func(context.Context) (int64, error)
|
||||
}{
|
||||
{"events", q.SweepExpiredEvents},
|
||||
{"audit_logs", q.SweepExpiredAuditLogs},
|
||||
{"audit_logs_org", q.SweepExpiredOrgAuditLogs},
|
||||
{"webhook_deliveries", q.SweepExpiredWebhookDeliveries},
|
||||
} {
|
||||
rows, err := sweep.fn(ctx)
|
||||
if err != nil {
|
||||
if ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
s.logger.Warn("retention sweep failed", "stream", sweep.name, "error", err)
|
||||
continue
|
||||
}
|
||||
if rows > 0 {
|
||||
s.logger.Info("retention sweep removed rows", "stream", sweep.name, "rows", rows)
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user