mirror of
https://github.com/Dvorinka/Primora.git
synced 2026-10-28 15:51:23 +00:00
feat: PostgREST-style document filtering and ordering
GET /collections/:id/documents accepts filter (field.op.value terms; eq neq gt gte lt lte like in is; dotted paths index into document data) and order (field.asc|desc). Whitelisted field/operator parsing with parameterized values; plain lists keep the sqlc path. CLI gains `documents list --collection --filter --order`; OpenAPI + generated client updated. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
1 parent
f9e7002048
commit
8892c6dc43
11 files changed
+489
-8
No files matched your search
@@ -12,6 +12,7 @@ import (
|
||||
"github.com/google/uuid"
|
||||
"github.com/jackc/pgx/v5"
|
||||
|
||||
db "github.com/tdvorak/primora/apps/backend/internal/database/db"
|
||||
"github.com/tdvorak/primora/apps/backend/internal/middleware"
|
||||
"github.com/tdvorak/primora/apps/backend/internal/observability"
|
||||
apperrors "github.com/tdvorak/primora/apps/backend/internal/response"
|
||||
@@ -1057,12 +1058,19 @@ func (h *HTTPHandler) listDocuments(c *gin.Context) {
|
||||
return
|
||||
}
|
||||
actor, _ := middleware.ActorFromContext(c)
|
||||
result, err := h.Platform.ListDocuments(c.Request.Context(), actor, collectionID, limit, offset)
|
||||
if err != nil {
|
||||
h.handleError(c, err)
|
||||
return
|
||||
var result []db.CoreDocument
|
||||
var total int64
|
||||
var err error
|
||||
// PostgREST-style filter/order switches to the dynamic-query path; the
|
||||
// plain list keeps using the sqlc queries it always has.
|
||||
if filter := c.Query("filter"); filter != "" || c.Query("order") != "" {
|
||||
result, total, err = h.Platform.SearchDocuments(c.Request.Context(), actor, collectionID, filter, c.Query("order"), limit, offset)
|
||||
} else {
|
||||
result, err = h.Platform.ListDocuments(c.Request.Context(), actor, collectionID, limit, offset)
|
||||
if err == nil {
|
||||
total, err = h.Platform.CountDocuments(c.Request.Context(), actor, collectionID)
|
||||
}
|
||||
}
|
||||
total, err := h.Platform.CountDocuments(c.Request.Context(), actor, collectionID)
|
||||
if err != nil {
|
||||
h.handleError(c, err)
|
||||
return
|
||||
|
||||
@@ -27,6 +27,12 @@ func (r *CoreRepository) Queries() *db.Queries {
|
||||
return r.queries
|
||||
}
|
||||
|
||||
// Pool exposes the pgx pool for dynamic queries sqlc can't express
|
||||
// (PostgREST-style document filtering builds its WHERE clause at runtime).
|
||||
func (r *CoreRepository) Pool() *pgxpool.Pool {
|
||||
return r.pool
|
||||
}
|
||||
|
||||
func (r *CoreRepository) WithTx(ctx context.Context, fn func(*db.Queries) error) error {
|
||||
tx, err := r.pool.BeginTx(ctx, pgx.TxOptions{})
|
||||
if err != nil {
|
||||
|
||||
@@ -0,0 +1,218 @@
|
||||
package services
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/google/uuid"
|
||||
|
||||
db "github.com/tdvorak/primora/apps/backend/internal/database/db"
|
||||
"github.com/tdvorak/primora/apps/backend/internal/models"
|
||||
)
|
||||
|
||||
// PostgREST-lite filtering over document JSONB:
|
||||
// ?filter=status.eq.active,age.gt.18,role.in.admin|editor
|
||||
// ?order=name.asc,created_at.desc
|
||||
// Values are always parameterized; only the whitelisted operators and
|
||||
// field-path syntax below ever reach the SQL string.
|
||||
|
||||
var docFieldRe = regexp.MustCompile(`^[A-Za-z_][A-Za-z0-9_]*$`)
|
||||
|
||||
// documentColumns are real table columns — everything else resolves into data.
|
||||
var documentColumns = map[string]string{
|
||||
"id": "id",
|
||||
"created_at": "created_at",
|
||||
"updated_at": "updated_at",
|
||||
}
|
||||
|
||||
// docFieldExpr maps a (possibly dotted) field path to a SQL text expression.
|
||||
// data #>> '{a,b}' extracts nested values as text — the correct comparison
|
||||
// domain for loosely-typed document fields.
|
||||
func docFieldExpr(field string) (string, error) {
|
||||
if col, ok := documentColumns[field]; ok {
|
||||
return col, nil
|
||||
}
|
||||
parts := strings.Split(field, ".")
|
||||
for _, p := range parts {
|
||||
if !docFieldRe.MatchString(p) {
|
||||
return "", fmt.Errorf("invalid field name %q — use [A-Za-z_][A-Za-z0-9_]* segments", field)
|
||||
}
|
||||
}
|
||||
return fmt.Sprintf("data #>> '{%s}'", strings.Join(parts, ",")), nil
|
||||
}
|
||||
|
||||
// numericExpr wraps the field for ordered comparisons — only used when the
|
||||
// filter value itself parses as a float, so the cast can never fail.
|
||||
func numericExpr(expr string) string {
|
||||
return fmt.Sprintf("(%s)::numeric", expr)
|
||||
}
|
||||
|
||||
// buildDocumentFilters turns "field.op.value" terms into parameterized SQL.
|
||||
// Returns the WHERE fragments and the argument list (already $N-numbered via
|
||||
// startArg). A fragment is empty for `is` filters since they carry no value.
|
||||
func buildDocumentFilters(filterParam string, startArg int) ([]string, []any, error) {
|
||||
var conds []string
|
||||
var args []any
|
||||
argN := startArg
|
||||
next := func(v any) string {
|
||||
args = append(args, v)
|
||||
argN++
|
||||
return fmt.Sprintf("$%d", argN-1)
|
||||
}
|
||||
|
||||
var docOps = []string{"eq", "neq", "gt", "gte", "lt", "lte", "like", "in", "is"}
|
||||
|
||||
for _, term := range strings.Split(filterParam, ",") {
|
||||
term = strings.TrimSpace(term)
|
||||
if term == "" {
|
||||
continue
|
||||
}
|
||||
// Field paths are themselves dotted, so the operator is located by
|
||||
// scanning for the first ".<known-op>." — the value may contain dots.
|
||||
var field, op, value string
|
||||
best := -1
|
||||
for _, candidate := range docOps {
|
||||
needle := "." + candidate + "."
|
||||
if i := strings.Index(term, needle); i > 0 && (best < 0 || i < best) {
|
||||
best = i
|
||||
field, op, value = term[:i], candidate, term[i+len(needle):]
|
||||
}
|
||||
}
|
||||
if field == "" {
|
||||
return nil, nil, fmt.Errorf("invalid filter %q — expected field.op.value (ops: %s)", term, strings.Join(docOps, ", "))
|
||||
}
|
||||
expr, err := docFieldExpr(field)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
_, isFloat := strconv.ParseFloat(value, 64)
|
||||
|
||||
switch op {
|
||||
case "eq", "neq":
|
||||
sym := "="
|
||||
if op == "neq" {
|
||||
sym = "<>"
|
||||
}
|
||||
conds = append(conds, fmt.Sprintf("%s %s %s", expr, sym, next(value)))
|
||||
case "gt", "gte", "lt", "lte":
|
||||
if isFloat != nil {
|
||||
return nil, nil, fmt.Errorf("filter %q: %s requires a numeric value", term, op)
|
||||
}
|
||||
sym := map[string]string{"gt": ">", "gte": ">=", "lt": "<", "lte": "<="}[op]
|
||||
f, _ := strconv.ParseFloat(value, 64)
|
||||
conds = append(conds, fmt.Sprintf("%s %s %s", numericExpr(expr), sym, next(f)))
|
||||
case "like":
|
||||
conds = append(conds, fmt.Sprintf("%s ILIKE %s", expr, next(value)))
|
||||
case "in":
|
||||
items := strings.Split(value, "|")
|
||||
for i, item := range items {
|
||||
items[i] = strings.TrimSpace(item)
|
||||
}
|
||||
conds = append(conds, fmt.Sprintf("%s = ANY(%s)", expr, next(items)))
|
||||
case "is":
|
||||
switch value {
|
||||
case "null":
|
||||
conds = append(conds, fmt.Sprintf("%s IS NULL", expr))
|
||||
case "true", "false":
|
||||
conds = append(conds, fmt.Sprintf("%s = %s", expr, next(value)))
|
||||
default:
|
||||
return nil, nil, fmt.Errorf("filter %q: is accepts null|true|false", term)
|
||||
}
|
||||
default:
|
||||
return nil, nil, fmt.Errorf("unknown operator %q in filter %q — supported: eq, neq, gt, gte, lt, lte, like, in, is", op, term)
|
||||
}
|
||||
}
|
||||
return conds, args, nil
|
||||
}
|
||||
|
||||
// buildDocumentOrder turns "name.asc,created_at.desc" into an ORDER BY list.
|
||||
func buildDocumentOrder(orderParam string) (string, error) {
|
||||
if orderParam == "" {
|
||||
return "created_at DESC", nil
|
||||
}
|
||||
var parts []string
|
||||
for _, term := range strings.Split(orderParam, ",") {
|
||||
term = strings.TrimSpace(term)
|
||||
if term == "" {
|
||||
continue
|
||||
}
|
||||
// Direction is a suffix, not a split — field paths are dotted too.
|
||||
field, dir := term, "asc"
|
||||
if strings.HasSuffix(term, ".desc") {
|
||||
field, dir = strings.TrimSuffix(term, ".desc"), "desc"
|
||||
} else if strings.HasSuffix(term, ".asc") {
|
||||
field = strings.TrimSuffix(term, ".asc")
|
||||
}
|
||||
expr, err := docFieldExpr(field)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
parts = append(parts, expr+" "+strings.ToUpper(dir))
|
||||
}
|
||||
if len(parts) == 0 {
|
||||
return "created_at DESC", nil
|
||||
}
|
||||
return strings.Join(parts, ", "), nil
|
||||
}
|
||||
|
||||
// SearchDocuments lists documents with PostgREST-style filter/order params.
|
||||
// Returns the page plus the un-paginated match count for "total".
|
||||
func (s *PlatformService) SearchDocuments(
|
||||
ctx context.Context,
|
||||
actor *models.Actor,
|
||||
collectionID uuid.UUID,
|
||||
filterParam, orderParam string,
|
||||
limit, offset int32,
|
||||
) ([]db.CoreDocument, int64, error) {
|
||||
collection, err := s.repo.Queries().GetCollectionByID(ctx, collectionID)
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
if err := s.requireProjectRole(ctx, actor, collection.ProjectID, "admin", "developer", "viewer"); err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
|
||||
args := []any{collectionID}
|
||||
conds, filterArgs, err := buildDocumentFilters(filterParam, len(args)+1)
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
args = append(args, filterArgs...)
|
||||
|
||||
where := "collection_id = $1"
|
||||
if len(conds) > 0 {
|
||||
where += " AND " + strings.Join(conds, " AND ")
|
||||
}
|
||||
order, err := buildDocumentOrder(orderParam)
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
|
||||
var total int64
|
||||
if err := s.repo.Pool().QueryRow(ctx,
|
||||
fmt.Sprintf("SELECT COUNT(*) FROM core.documents WHERE %s", where), args...).Scan(&total); err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
|
||||
args = append(args, limit, offset)
|
||||
rows, err := s.repo.Pool().Query(ctx, fmt.Sprintf(
|
||||
"SELECT * FROM core.documents WHERE %s ORDER BY %s LIMIT $%d OFFSET $%d",
|
||||
where, order, len(args)-1, len(args)), args...)
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var out []db.CoreDocument
|
||||
for rows.Next() {
|
||||
var d db.CoreDocument
|
||||
if err := rows.Scan(&d.ID, &d.CollectionID, &d.Data, &d.CreatedByUserID, &d.CreatedAt, &d.UpdatedAt); err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
out = append(out, d)
|
||||
}
|
||||
return out, total, rows.Err()
|
||||
}
|
||||
@@ -0,0 +1,119 @@
|
||||
package services
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestBuildDocumentFilters(t *testing.T) {
|
||||
t.Run("eq on a nested path uses #>>", func(t *testing.T) {
|
||||
conds, args, err := buildDocumentFilters("meta.city.eq.Prague", 2)
|
||||
if err != nil {
|
||||
t.Fatalf("err: %v", err)
|
||||
}
|
||||
want := "data #>> '{meta,city}' = $2"
|
||||
if conds[0] != want || !reflect.DeepEqual(args, []any{"Prague"}) {
|
||||
t.Fatalf("got %v %v", conds, args)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("numeric ops cast to ::numeric", func(t *testing.T) {
|
||||
conds, args, err := buildDocumentFilters("age.gt.18,age.lt.65", 2)
|
||||
if err != nil {
|
||||
t.Fatalf("err: %v", err)
|
||||
}
|
||||
if !strings.Contains(conds[0], "::numeric > $2") || !strings.Contains(conds[1], "::numeric < $3") {
|
||||
t.Fatalf("got %v", conds)
|
||||
}
|
||||
if args[0] != 18.0 || args[1] != 65.0 {
|
||||
t.Fatalf("args: %v", args)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("numeric op rejects non-numeric value", func(t *testing.T) {
|
||||
_, _, err := buildDocumentFilters("age.gt.abc", 2)
|
||||
if err == nil {
|
||||
t.Fatal("expected error")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("like becomes ILIKE", func(t *testing.T) {
|
||||
conds, _, err := buildDocumentFilters("name.like.%_smith%", 2)
|
||||
if err != nil || conds[0] != "data #>> '{name}' ILIKE $2" {
|
||||
t.Fatalf("got %v %v", conds, err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("in becomes ANY with a text array", func(t *testing.T) {
|
||||
conds, args, err := buildDocumentFilters("role.in.admin|editor", 2)
|
||||
if err != nil || !strings.Contains(conds[0], "= ANY($2)") {
|
||||
t.Fatalf("got %v", conds)
|
||||
}
|
||||
if !reflect.DeepEqual(args[0], []string{"admin", "editor"}) {
|
||||
t.Fatalf("args: %v", args)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("is null carries no argument", func(t *testing.T) {
|
||||
conds, args, err := buildDocumentFilters("deleted_at.is.null", 2)
|
||||
if err != nil || len(args) != 0 || conds[0] != "data #>> '{deleted_at}' IS NULL" {
|
||||
t.Fatalf("got %v %v", conds, args)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("rejects injection-shaped field", func(t *testing.T) {
|
||||
_, _, err := buildDocumentFilters("x' OR '1'='1.eq.a", 2)
|
||||
if err == nil {
|
||||
t.Fatal("expected error")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("rejects term without a known operator", func(t *testing.T) {
|
||||
_, _, err := buildDocumentFilters("age.between.5", 2)
|
||||
if err == nil || !strings.Contains(err.Error(), "invalid filter") {
|
||||
t.Fatalf("got %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("malformed term errors", func(t *testing.T) {
|
||||
_, _, err := buildDocumentFilters("justafield", 2)
|
||||
if err == nil {
|
||||
t.Fatal("expected error")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("real columns resolve directly", func(t *testing.T) {
|
||||
conds, _, err := buildDocumentFilters("created_at.gt.2026-01-01", 2)
|
||||
// created_at is a column — but value isn't numeric so gt must fail
|
||||
if err == nil {
|
||||
t.Fatalf("expected numeric-required error, got %v", conds)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestBuildDocumentOrder(t *testing.T) {
|
||||
cases := map[string]string{
|
||||
"": "created_at DESC",
|
||||
"name.asc": "data #>> '{name}' ASC",
|
||||
"name": "data #>> '{name}' ASC",
|
||||
"created_at.desc": "created_at DESC",
|
||||
"a.b.asc,id.desc": "data #>> '{a,b}' ASC, id DESC",
|
||||
}
|
||||
for in, want := range cases {
|
||||
got, err := buildDocumentOrder(in)
|
||||
if err != nil {
|
||||
t.Fatalf("%q: %v", in, err)
|
||||
}
|
||||
if got != want {
|
||||
t.Errorf("%q: got %q want %q", in, got, want)
|
||||
}
|
||||
}
|
||||
// A non-asc/desc suffix is a nested field path, not a direction error.
|
||||
if got, err := buildDocumentOrder("name.sideways"); err != nil || got != "data #>> '{name,sideways}' ASC" {
|
||||
t.Errorf("nested order path: got %q err %v", got, err)
|
||||
}
|
||||
if _, err := buildDocumentOrder("x;drop.asc"); err == nil {
|
||||
t.Error("expected field error")
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user