mirror of
https://github.com/Dvorinka/Primora.git
synced 2026-10-28 07:41:23 +00:00
feat: Presigned object URLs (s3 driver)
POST /buckets/:id/object-presigns mints SigV4 query-signed GET/PUT URLs so clients move bytes directly against S3 — large transfers skip the backend entirely. - storage.Presigner optional interface; S3Store implements, local driver returns a clear error (no external surface to sign). - TTL 60s–1h (default 15m); download requires read role + existing object row (no key probing); upload requires write role. - S3_PUBLIC_ENDPOINT rewrites the client-facing host for internal S3_ENDPOINTs — signatures bind the host, URLs are valid only where minted. - CLI: primora objects:presign <bucket> <key> [--upload] [--ttl]. Verified live against cloudserver: GET fetched the object with no auth headers; PUT pushed 1 MB direct to S3; unsigned list → 403; untracked-key download presign → 404. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
1 parent
a84049b8fc
commit
fd36389328
22 files changed
+446
-7
No files matched your search
@@ -28,7 +28,11 @@ type S3Config struct {
|
||||
SecretAccessKey string
|
||||
Prefix string // optional key prefix, e.g. "primora/"
|
||||
PathStyle bool // bucket/key path style (MinIO, Garage) vs virtual-hosted
|
||||
HTTPClient *http.Client
|
||||
// PublicEndpoint rewrites presigned-URL hosts for clients that can't
|
||||
// reach Endpoint (e.g. minio:9000 inside compose). Signature binds the
|
||||
// public host — the URL is only valid there.
|
||||
PublicEndpoint string
|
||||
HTTPClient *http.Client
|
||||
}
|
||||
|
||||
// S3Store implements Store against an S3-compatible API using stdlib-only
|
||||
@@ -271,6 +275,88 @@ func (s *S3Store) DeleteBucket(bucketID string) error {
|
||||
}
|
||||
}
|
||||
|
||||
// Presign mints a SigV4 query-signed URL for a single GET/PUT on the
|
||||
// object — clients talk to S3 directly instead of proxying bytes through
|
||||
// the backend. TTL is capped at one hour.
|
||||
func (s *S3Store) Presign(ctx context.Context, bucketID, objectKey, method string, ttl time.Duration) (*PresignedURL, error) {
|
||||
if method != http.MethodGet && method != http.MethodPut {
|
||||
return nil, fmt.Errorf("s3 presign: unsupported method %q", method)
|
||||
}
|
||||
key, err := s.objectKey(bucketID, objectKey)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if ttl <= 0 {
|
||||
ttl = 15 * time.Minute
|
||||
}
|
||||
if ttl > time.Hour {
|
||||
ttl = time.Hour
|
||||
}
|
||||
|
||||
now := s.now().UTC()
|
||||
amzDate := now.Format("20060102T150405Z")
|
||||
dateStamp := now.Format("20060102")
|
||||
scope := fmt.Sprintf("%s/%s/s3/aws4_request", dateStamp, s.cfg.Region)
|
||||
|
||||
// Build the unsigned URL — same path/vhost logic as signedRequest, but
|
||||
// on the client-facing endpoint when one is configured.
|
||||
var base *url.URL
|
||||
endpoint := strings.TrimSuffix(s.cfg.PublicEndpoint, "/")
|
||||
if endpoint == "" {
|
||||
endpoint = strings.TrimSuffix(s.cfg.Endpoint, "/")
|
||||
}
|
||||
if s.cfg.PathStyle {
|
||||
u, err := url.Parse(endpoint + "/" + s.cfg.Bucket + "/" + escapeKey(key))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("s3 presign url: %w", err)
|
||||
}
|
||||
base = u
|
||||
} else {
|
||||
u, err := url.Parse(endpoint)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("s3 presign url: %w", err)
|
||||
}
|
||||
u.Host = s.cfg.Bucket + "." + u.Host
|
||||
u.Path = "/" + escapeKey(key)
|
||||
base = u
|
||||
}
|
||||
|
||||
q := base.Query()
|
||||
q.Set("X-Amz-Algorithm", "AWS4-HMAC-SHA256")
|
||||
q.Set("X-Amz-Credential", s.cfg.AccessKeyID+"/"+scope)
|
||||
q.Set("X-Amz-Date", amzDate)
|
||||
q.Set("X-Amz-Expires", fmt.Sprintf("%d", int64(ttl.Seconds())))
|
||||
q.Set("X-Amz-SignedHeaders", "host")
|
||||
|
||||
canonicalURI := base.EscapedPath()
|
||||
if canonicalURI == "" {
|
||||
canonicalURI = "/"
|
||||
}
|
||||
canonicalRequest := strings.Join([]string{
|
||||
method,
|
||||
canonicalURI,
|
||||
canonicalQuery(q),
|
||||
"host:" + base.Host + "\n",
|
||||
"host",
|
||||
"UNSIGNED-PAYLOAD",
|
||||
}, "\n")
|
||||
stringToSign := strings.Join([]string{
|
||||
"AWS4-HMAC-SHA256",
|
||||
amzDate,
|
||||
scope,
|
||||
hexSHA256(canonicalRequest),
|
||||
}, "\n")
|
||||
|
||||
kDate := hmacSHA256([]byte("AWS4"+s.cfg.SecretAccessKey), dateStamp)
|
||||
kRegion := hmacSHA256(kDate, s.cfg.Region)
|
||||
kService := hmacSHA256(kRegion, "s3")
|
||||
kSigning := hmacSHA256(kService, "aws4_request")
|
||||
q.Set("X-Amz-Signature", hex.EncodeToString(hmacSHA256(kSigning, stringToSign)))
|
||||
base.RawQuery = q.Encode()
|
||||
|
||||
return &PresignedURL{URL: base.String(), Method: method, ExpiresAt: now.Add(ttl)}, nil
|
||||
}
|
||||
|
||||
type listResult struct {
|
||||
Contents []struct{ Key string `xml:"Key"` } `xml:"Contents"`
|
||||
NextContinuationToken string `xml:"NextContinuationToken"`
|
||||
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
@@ -214,3 +215,37 @@ func TestCanonicalQueryRFC3986(t *testing.T) {
|
||||
t.Fatalf("canonicalQuery = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestS3StorePresign(t *testing.T) {
|
||||
store, _ := newTestS3(t)
|
||||
ctx := context.Background()
|
||||
|
||||
dl, err := store.Presign(ctx, "bucket-a", "docs/x.txt", http.MethodGet, 10*time.Minute)
|
||||
if err != nil {
|
||||
t.Fatalf("Presign: %v", err)
|
||||
}
|
||||
if dl.Method != http.MethodGet || dl.ExpiresAt.IsZero() {
|
||||
t.Fatalf("bad presign: %+v", dl)
|
||||
}
|
||||
u, err := url.Parse(dl.URL)
|
||||
if err != nil {
|
||||
t.Fatalf("url parse: %v", err)
|
||||
}
|
||||
q := u.Query()
|
||||
for _, k := range []string{"X-Amz-Algorithm", "X-Amz-Credential", "X-Amz-Date", "X-Amz-Expires", "X-Amz-SignedHeaders", "X-Amz-Signature"} {
|
||||
if q.Get(k) == "" {
|
||||
t.Fatalf("missing %s in %s", k, dl.URL)
|
||||
}
|
||||
}
|
||||
if q.Get("X-Amz-Expires") != "600" || q.Get("X-Amz-SignedHeaders") != "host" {
|
||||
t.Fatalf("bad query: %s", dl.URL)
|
||||
}
|
||||
if _, err := store.Presign(ctx, "b", "k", http.MethodDelete, 0); err == nil {
|
||||
t.Fatal("expected method rejection")
|
||||
}
|
||||
// TTL cap.
|
||||
capped, _ := store.Presign(ctx, "b", "k.txt", http.MethodPut, 48*time.Hour)
|
||||
if cu, _ := url.Parse(capped.URL); cu.Query().Get("X-Amz-Expires") != "3600" {
|
||||
t.Fatalf("ttl not capped: %s", capped.URL)
|
||||
}
|
||||
}
|
||||
@@ -3,6 +3,7 @@ package storage
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Store is the object-storage contract the platform service depends on.
|
||||
@@ -14,3 +15,17 @@ type Store interface {
|
||||
MoveBetweenBuckets(sourceBucketID, destinationBucketID, fromKey, toKey string) (string, error)
|
||||
DeleteBucket(bucketID string) error
|
||||
}
|
||||
|
||||
// PresignedURL is a short-lived direct-to-storage URL for one operation.
|
||||
type PresignedURL struct {
|
||||
URL string `json:"url"`
|
||||
Method string `json:"method"`
|
||||
ExpiresAt time.Time `json:"expires_at"`
|
||||
}
|
||||
|
||||
// Presigner is an optional capability: stores backed by a real object API
|
||||
// (S3) can mint URLs clients call directly. The local driver can't — its
|
||||
// "URL" would have to be served by this backend anyway.
|
||||
type Presigner interface {
|
||||
Presign(ctx context.Context, bucketID, objectKey, method string, ttl time.Duration) (*PresignedURL, error)
|
||||
}
|
||||
Reference in new issue
Block a user