mirror of
https://github.com/Dvorinka/Containr.git
synced 2026-10-28 15:51:26 +00:00
feat(databases): S3-compatible backup targets — offsite archives + remote restore
Phase D3. Managed databases can ship backup archives to a configured S3-compatible destination (AWS S3, MinIO, Garage, …) while the local copy remains intact. - backup_targets table: owner-scoped endpoint/bucket/region/prefix + encrypted access/secret keys (enc:v1:, never returned, has_credentials) - create/update probe bucket access before storing; empty keys on update keep stored credentials; delete refused (409) while referenced - databases.backup_target_id assigns a target; external DBs rejected - backup pipeline uploads via a holder container over docker cp streams; remote failure keeps the local archive (logged), row gets remote:true - restore prefers the local archive; on a missing local copy it downloads the remote object into the backup volume first - holder containers must be started before archive ops — docker cp does not resolve volume mounts on never-started (Created) containers, and removing the source mid-stream stalls the reader without EOF - CLI: backup-targets list|add|update|remove|test + databases update --backup-target; MCP: 5 new tools; Settings UI card + per-database target picker + offsite badge on backups; OpenAPI + llms.txt + CLI.md Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
1 parent
b7131965b1
commit
6f43bbf8f6
25 files changed
+1901
-32
No files matched your search
@@ -0,0 +1,506 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"bytes"
|
||||
"context"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"containr/internal/database/sqlcdb"
|
||||
"containr/internal/docker"
|
||||
"containr/internal/secrets"
|
||||
|
||||
"github.com/docker/docker/api/types/mount"
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/google/uuid"
|
||||
"github.com/minio/minio-go/v7"
|
||||
"github.com/minio/minio-go/v7/pkg/credentials"
|
||||
)
|
||||
|
||||
// BackupTarget is an S3-compatible archive destination. Secret keys are
|
||||
// stored encrypted and never serialized — has_credentials signals presence.
|
||||
type BackupTarget struct {
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
Endpoint string `json:"endpoint"`
|
||||
Bucket string `json:"bucket"`
|
||||
Region string `json:"region,omitempty"`
|
||||
Prefix string `json:"prefix,omitempty"`
|
||||
HasCredentials bool `json:"has_credentials"`
|
||||
UseTLS bool `json:"use_tls"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
UpdatedAt time.Time `json:"updated_at"`
|
||||
|
||||
accessKey string `json:"-"`
|
||||
secretKey string `json:"-"`
|
||||
}
|
||||
|
||||
type backupTargetRequest struct {
|
||||
Name string `json:"name"`
|
||||
Endpoint string `json:"endpoint" binding:"required"`
|
||||
Bucket string `json:"bucket" binding:"required"`
|
||||
Region string `json:"region"`
|
||||
Prefix string `json:"prefix"`
|
||||
AccessKey string `json:"access_key"`
|
||||
SecretKey string `json:"secret_key"`
|
||||
UseTLS *bool `json:"use_tls"`
|
||||
}
|
||||
|
||||
func mapBackupTargetRow(row sqlcdb.BackupTarget) BackupTarget {
|
||||
return BackupTarget{
|
||||
ID: row.ID,
|
||||
Name: row.Name,
|
||||
Endpoint: row.Endpoint,
|
||||
Bucket: row.Bucket,
|
||||
Region: row.Region,
|
||||
Prefix: row.Prefix,
|
||||
HasCredentials: row.SecretKey != "",
|
||||
UseTLS: row.UseTls,
|
||||
CreatedAt: databaseNullTime(row.CreatedAt),
|
||||
UpdatedAt: databaseNullTime(row.UpdatedAt),
|
||||
accessKey: secrets.Decrypt(row.AccessKey),
|
||||
secretKey: secrets.Decrypt(row.SecretKey),
|
||||
}
|
||||
}
|
||||
|
||||
// s3Client builds a client for the target. Endpoint is a bare host[:port] —
|
||||
// scheme comes from use_tls, matching MinIO/Garage/AWS conventions.
|
||||
func (t BackupTarget) s3Client() (*minio.Client, error) {
|
||||
opts := &minio.Options{Secure: t.UseTLS, Region: t.Region}
|
||||
if t.accessKey != "" || t.secretKey != "" {
|
||||
opts.Creds = credentials.NewStaticV4(t.accessKey, t.secretKey, "")
|
||||
} else {
|
||||
opts.Creds = credentials.NewIAM("")
|
||||
}
|
||||
return minio.New(t.Endpoint, opts)
|
||||
}
|
||||
|
||||
func normalizeBackupTargetRequest(req *backupTargetRequest) error {
|
||||
req.Name = strings.TrimSpace(req.Name)
|
||||
req.Endpoint = strings.TrimSpace(req.Endpoint)
|
||||
req.Endpoint = strings.TrimPrefix(strings.TrimPrefix(req.Endpoint, "https://"), "http://")
|
||||
req.Endpoint = strings.TrimSuffix(req.Endpoint, "/")
|
||||
req.Bucket = strings.TrimSpace(req.Bucket)
|
||||
req.Prefix = strings.Trim(strings.TrimSpace(req.Prefix), "/")
|
||||
if req.Name == "" {
|
||||
return errors.New("name is required")
|
||||
}
|
||||
if req.Endpoint == "" || req.Bucket == "" {
|
||||
return errors.New("endpoint and bucket are required")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// probeBackupTarget verifies credentials and bucket access.
|
||||
func probeBackupTarget(ctx context.Context, t BackupTarget) (time.Duration, error) {
|
||||
client, err := t.s3Client()
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
start := time.Now()
|
||||
exists, err := client.BucketExists(ctx, t.Bucket)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if !exists {
|
||||
return 0, fmt.Errorf("bucket %q not found or not accessible", t.Bucket)
|
||||
}
|
||||
return time.Since(start), nil
|
||||
}
|
||||
|
||||
func (h *DatabaseHandler) ListBackupTargets(c *gin.Context) {
|
||||
userID, ok := requireAuthenticatedUserID(c)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
rows, err := h.queries.ListBackupTargetsByUser(c.Request.Context(), userID)
|
||||
if err != nil {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to list backup targets"})
|
||||
return
|
||||
}
|
||||
targets := make([]BackupTarget, 0, len(rows))
|
||||
for _, row := range rows {
|
||||
t := mapBackupTargetRow(row)
|
||||
t.accessKey, t.secretKey = "", ""
|
||||
targets = append(targets, t)
|
||||
}
|
||||
c.JSON(http.StatusOK, gin.H{"backup_targets": targets})
|
||||
}
|
||||
|
||||
func (h *DatabaseHandler) CreateBackupTarget(c *gin.Context) {
|
||||
userID, ok := requireAuthenticatedUserID(c)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
var req backupTargetRequest
|
||||
if err := c.ShouldBindJSON(&req); err != nil {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": err.Error(), "code": "VALIDATION"})
|
||||
return
|
||||
}
|
||||
if err := normalizeBackupTargetRequest(&req); err != nil {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": err.Error(), "code": "VALIDATION"})
|
||||
return
|
||||
}
|
||||
|
||||
useTLS := true
|
||||
if req.UseTLS != nil {
|
||||
useTLS = *req.UseTLS
|
||||
}
|
||||
target := BackupTarget{
|
||||
ID: "bt_" + strings.ReplaceAll(uuid.NewString(), "-", "")[:12],
|
||||
Name: req.Name,
|
||||
Endpoint: req.Endpoint,
|
||||
Bucket: req.Bucket,
|
||||
Region: req.Region,
|
||||
Prefix: req.Prefix,
|
||||
UseTLS: useTLS,
|
||||
accessKey: req.AccessKey,
|
||||
secretKey: req.SecretKey,
|
||||
}
|
||||
|
||||
probeCtx, cancel := context.WithTimeout(c.Request.Context(), 10*time.Second)
|
||||
defer cancel()
|
||||
if _, err := probeBackupTarget(probeCtx, target); err != nil {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": fmt.Sprintf("Connection check failed: %v", err), "code": "DEPENDENCY_UNAVAILABLE"})
|
||||
return
|
||||
}
|
||||
|
||||
enc := func(v string) string {
|
||||
if v == "" {
|
||||
return ""
|
||||
}
|
||||
return secrets.Encrypt(v)
|
||||
}
|
||||
now := sql.NullTime{Time: time.Now(), Valid: true}
|
||||
if err := h.queries.CreateBackupTarget(c.Request.Context(), sqlcdb.CreateBackupTargetParams{
|
||||
ID: target.ID,
|
||||
UserID: userID,
|
||||
Name: target.Name,
|
||||
Endpoint: target.Endpoint,
|
||||
Bucket: target.Bucket,
|
||||
Region: target.Region,
|
||||
Prefix: target.Prefix,
|
||||
AccessKey: enc(req.AccessKey),
|
||||
SecretKey: enc(req.SecretKey),
|
||||
UseTls: useTLS,
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
}); err != nil {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to create backup target"})
|
||||
return
|
||||
}
|
||||
LogAudit(userID, "backup_target", target.ID, "create", map[string]interface{}{"name": req.Name, "endpoint": req.Endpoint, "bucket": req.Bucket})
|
||||
target.accessKey, target.secretKey = "", ""
|
||||
target.HasCredentials = req.SecretKey != ""
|
||||
target.CreatedAt, target.UpdatedAt = now.Time, now.Time
|
||||
c.JSON(http.StatusCreated, gin.H{"backup_target": target})
|
||||
}
|
||||
|
||||
func (h *DatabaseHandler) UpdateBackupTarget(c *gin.Context) {
|
||||
userID, ok := requireAuthenticatedUserID(c)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
targetID := c.Param("id")
|
||||
row, err := h.queries.GetBackupTargetByIDAndUser(c.Request.Context(), sqlcdb.GetBackupTargetByIDAndUserParams{ID: targetID, UserID: userID})
|
||||
if err != nil {
|
||||
c.JSON(http.StatusNotFound, gin.H{"error": "Backup target not found", "code": "NOT_FOUND"})
|
||||
return
|
||||
}
|
||||
var req backupTargetRequest
|
||||
if err := c.ShouldBindJSON(&req); err != nil {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": err.Error(), "code": "VALIDATION"})
|
||||
return
|
||||
}
|
||||
if err := normalizeBackupTargetRequest(&req); err != nil {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": err.Error(), "code": "VALIDATION"})
|
||||
return
|
||||
}
|
||||
useTLS := row.UseTls
|
||||
if req.UseTLS != nil {
|
||||
useTLS = *req.UseTLS
|
||||
}
|
||||
// Empty credentials keep the stored pair.
|
||||
accessKey := row.AccessKey
|
||||
secretKey := row.SecretKey
|
||||
if req.AccessKey != "" || req.SecretKey != "" {
|
||||
accessKey = secrets.Encrypt(req.AccessKey)
|
||||
secretKey = secrets.Encrypt(req.SecretKey)
|
||||
}
|
||||
|
||||
target := BackupTarget{
|
||||
Endpoint: req.Endpoint,
|
||||
Bucket: req.Bucket,
|
||||
Region: req.Region,
|
||||
Prefix: req.Prefix,
|
||||
UseTLS: useTLS,
|
||||
accessKey: secrets.Decrypt(accessKey),
|
||||
secretKey: secrets.Decrypt(secretKey),
|
||||
}
|
||||
probeCtx, cancel := context.WithTimeout(c.Request.Context(), 10*time.Second)
|
||||
defer cancel()
|
||||
if _, err := probeBackupTarget(probeCtx, target); err != nil {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": fmt.Sprintf("Connection check failed: %v", err), "code": "DEPENDENCY_UNAVAILABLE"})
|
||||
return
|
||||
}
|
||||
|
||||
if err := h.queries.UpdateBackupTargetByIDAndUser(c.Request.Context(), sqlcdb.UpdateBackupTargetByIDAndUserParams{
|
||||
Name: req.Name,
|
||||
Endpoint: req.Endpoint,
|
||||
Bucket: req.Bucket,
|
||||
Region: req.Region,
|
||||
Prefix: req.Prefix,
|
||||
AccessKey: accessKey,
|
||||
SecretKey: secretKey,
|
||||
UseTls: useTLS,
|
||||
UpdatedAt: sql.NullTime{Time: time.Now(), Valid: true},
|
||||
ID: targetID,
|
||||
UserID: userID,
|
||||
}); err != nil {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to update backup target"})
|
||||
return
|
||||
}
|
||||
c.JSON(http.StatusOK, gin.H{"message": "Backup target updated"})
|
||||
}
|
||||
|
||||
func (h *DatabaseHandler) DeleteBackupTarget(c *gin.Context) {
|
||||
userID, ok := requireAuthenticatedUserID(c)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
targetID := c.Param("id")
|
||||
count, err := h.queries.CountDatabasesUsingBackupTarget(c.Request.Context(), sql.NullString{String: targetID, Valid: true})
|
||||
if err != nil {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to check target usage"})
|
||||
return
|
||||
}
|
||||
if count > 0 {
|
||||
c.JSON(http.StatusConflict, gin.H{"error": fmt.Sprintf("%d database(s) still back up to this target", count), "code": "CONFLICT"})
|
||||
return
|
||||
}
|
||||
if err := h.queries.DeleteBackupTargetByIDAndUser(c.Request.Context(), sqlcdb.DeleteBackupTargetByIDAndUserParams{ID: targetID, UserID: userID}); err != nil {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to delete backup target"})
|
||||
return
|
||||
}
|
||||
LogAudit(userID, "backup_target", targetID, "delete", nil)
|
||||
c.JSON(http.StatusOK, gin.H{"message": "Backup target deleted"})
|
||||
}
|
||||
|
||||
func (h *DatabaseHandler) TestBackupTarget(c *gin.Context) {
|
||||
userID, ok := requireAuthenticatedUserID(c)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
targetID := c.Param("id")
|
||||
row, err := h.queries.GetBackupTargetByIDAndUser(c.Request.Context(), sqlcdb.GetBackupTargetByIDAndUserParams{ID: targetID, UserID: userID})
|
||||
if err != nil {
|
||||
c.JSON(http.StatusNotFound, gin.H{"error": "Backup target not found", "code": "NOT_FOUND"})
|
||||
return
|
||||
}
|
||||
target := mapBackupTargetRow(row)
|
||||
probeCtx, cancel := context.WithTimeout(c.Request.Context(), 10*time.Second)
|
||||
defer cancel()
|
||||
latency, probeErr := probeBackupTarget(probeCtx, target)
|
||||
if probeErr != nil {
|
||||
c.JSON(http.StatusOK, gin.H{"ok": false, "error": probeErr.Error()})
|
||||
return
|
||||
}
|
||||
c.JSON(http.StatusOK, gin.H{"ok": true, "latency_ms": latency.Milliseconds()})
|
||||
}
|
||||
|
||||
// uploadBackupToTarget copies the archive out of the backup volume and puts
|
||||
// it in the configured S3 target. Returns the object key, or "" when no
|
||||
// target is configured.
|
||||
func (h *DatabaseHandler) uploadBackupToTarget(ctx context.Context, db sqlcdb.DatabaseService, archivePath, backupID string) (string, error) {
|
||||
if !db.BackupTargetID.Valid || db.BackupTargetID.String == "" {
|
||||
return "", nil
|
||||
}
|
||||
targetRow, err := h.queries.GetBackupTargetByID(ctx, db.BackupTargetID.String)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("backup target lookup failed: %w", err)
|
||||
}
|
||||
target := mapBackupTargetRow(targetRow)
|
||||
|
||||
archive, err := h.readBackupArchive(ctx, archivePath)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("read archive: %w", err)
|
||||
}
|
||||
defer func() {
|
||||
_, _ = io.Copy(io.Discard, archive)
|
||||
_ = archive.Close()
|
||||
}()
|
||||
|
||||
client, err := target.s3Client()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
key := target.Prefix
|
||||
if key != "" {
|
||||
key += "/"
|
||||
}
|
||||
key += "databases/" + db.ID + "/" + sanitizeBackupArchivePath(archivePath)
|
||||
_, err = client.PutObject(ctx, target.Bucket, key, archive, -1, minio.PutObjectOptions{ContentType: "application/gzip"})
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("s3 upload: %w", err)
|
||||
}
|
||||
_ = h.queries.SetDatabaseBackupRemoteKeyByID(ctx, sqlcdb.SetDatabaseBackupRemoteKeyByIDParams{
|
||||
RemoteKey: sql.NullString{String: key, Valid: true},
|
||||
ID: backupID,
|
||||
})
|
||||
return key, nil
|
||||
}
|
||||
|
||||
// readBackupArchive streams an archive out of the shared backup volume via a
|
||||
// stopped holder container — the volume has no direct host path.
|
||||
func (h *DatabaseHandler) readBackupArchive(ctx context.Context, archivePath string) (io.ReadCloser, error) {
|
||||
holderID, err := h.createBackupVolumeHolder(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
reader, err := h.dockerClient.CopyFromContainer(ctx, holderID, "/backup/"+sanitizeBackupArchivePath(archivePath))
|
||||
if err != nil {
|
||||
_ = h.dockerClient.RemoveContainer(context.Background(), holderID, true)
|
||||
return nil, err
|
||||
}
|
||||
// The holder must outlive the stream — deleting the source container while
|
||||
// dockerd serves the archive stalls the read without an error.
|
||||
return &archiveReader{tar: reader, cleanup: func() {
|
||||
_ = h.dockerClient.RemoveContainer(context.Background(), holderID, true)
|
||||
}}, nil
|
||||
}
|
||||
|
||||
// downloadBackupToVolume fetches a remote archive back into the backup volume
|
||||
// so the normal restore path can run unchanged.
|
||||
func (h *DatabaseHandler) downloadBackupToVolume(ctx context.Context, db sqlcdb.DatabaseService, remoteKey, archivePath string) error {
|
||||
targetRow, err := h.queries.GetBackupTargetByID(ctx, db.BackupTargetID.String)
|
||||
if err != nil {
|
||||
return fmt.Errorf("backup target lookup failed: %w", err)
|
||||
}
|
||||
target := mapBackupTargetRow(targetRow)
|
||||
client, err := target.s3Client()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
obj, err := client.GetObject(ctx, target.Bucket, remoteKey, minio.GetObjectOptions{})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer obj.Close()
|
||||
|
||||
archive := sanitizeBackupArchivePath(archivePath)
|
||||
if archive == "" {
|
||||
return errors.New("backup path is required")
|
||||
}
|
||||
|
||||
var buf bytes.Buffer
|
||||
tw := tar.NewWriter(&buf)
|
||||
data, err := io.ReadAll(obj)
|
||||
if err != nil {
|
||||
return fmt.Errorf("s3 download: %w", err)
|
||||
}
|
||||
if err := tw.WriteHeader(&tar.Header{Name: archive, Mode: 0o644, Size: int64(len(data))}); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := tw.Write(data); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := tw.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
holderID, err := h.createBackupVolumeHolder(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() { _ = h.dockerClient.RemoveContainer(context.Background(), holderID, true) }()
|
||||
return h.dockerClient.CopyToContainer(ctx, holderID, "/backup", &buf)
|
||||
}
|
||||
|
||||
// backupArchiveExists reports whether the archive is still in the local
|
||||
// backup volume — a copy attempt on a holder container is the cheapest check.
|
||||
func (h *DatabaseHandler) backupArchiveExists(ctx context.Context, archivePath string) (bool, error) {
|
||||
holderID, err := h.createBackupVolumeHolder(ctx)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
defer func() { _ = h.dockerClient.RemoveContainer(context.Background(), holderID, true) }()
|
||||
reader, err := h.dockerClient.CopyFromContainer(ctx, holderID, "/backup/"+sanitizeBackupArchivePath(archivePath))
|
||||
if err != nil {
|
||||
return false, nil
|
||||
}
|
||||
_ = reader.Close()
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// createBackupVolumeHolder creates a container mounting the backup volume so
|
||||
// archives can be copied in and out. It must be started — docker cp/archive
|
||||
// only resolves volume mounts on containers that have run at least once.
|
||||
func (h *DatabaseHandler) createBackupVolumeHolder(ctx context.Context) (string, error) {
|
||||
if err := h.ensureImage(ctx, managedDatabaseBackupImage); err != nil {
|
||||
return "", err
|
||||
}
|
||||
holderID, err := h.dockerClient.CreateContainer(ctx, docker.ContainerConfig{
|
||||
Image: managedDatabaseBackupImage,
|
||||
Cmd: []string{"true"},
|
||||
RestartPolicy: "no",
|
||||
Mounts: []mount.Mount{{Type: mount.TypeVolume, Source: managedDatabaseBackupVolume, Target: "/backup"}},
|
||||
Labels: map[string]string{"containr.managed": "true", "containr.utility": "backup-io"},
|
||||
NetworkMode: "none",
|
||||
})
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if err := h.dockerClient.StartContainer(ctx, holderID); err != nil {
|
||||
_ = h.dockerClient.RemoveContainer(context.Background(), holderID, true)
|
||||
return "", fmt.Errorf("failed to start backup holder: %w", err)
|
||||
}
|
||||
return holderID, nil
|
||||
}
|
||||
|
||||
// archiveReader unwraps the tar stream CopyFromContainer returns.
|
||||
type archiveReader struct {
|
||||
tar io.ReadCloser
|
||||
tr *tar.Reader
|
||||
err error
|
||||
cleanup func()
|
||||
}
|
||||
|
||||
func (r *archiveReader) Read(p []byte) (int, error) {
|
||||
if r.err != nil {
|
||||
return 0, r.err
|
||||
}
|
||||
if r.tr == nil {
|
||||
r.tr = tar.NewReader(r.tar)
|
||||
// Skip to the first regular file entry.
|
||||
for {
|
||||
hdr, err := r.tr.Next()
|
||||
if err != nil {
|
||||
r.err = err
|
||||
return 0, err
|
||||
}
|
||||
if hdr.Typeflag == tar.TypeReg {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
n, err := r.tr.Read(p)
|
||||
if err != nil {
|
||||
r.err = err
|
||||
}
|
||||
return n, err
|
||||
}
|
||||
|
||||
func (r *archiveReader) Close() error {
|
||||
err := r.tar.Close()
|
||||
if r.cleanup != nil {
|
||||
r.cleanup()
|
||||
r.cleanup = nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
@@ -0,0 +1,60 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"bytes"
|
||||
"io"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestNormalizeBackupTargetRequest(t *testing.T) {
|
||||
req := backupTargetRequest{
|
||||
Name: " garage ",
|
||||
Endpoint: "https://garage.local:3900/",
|
||||
Bucket: " backups ",
|
||||
Prefix: " /prod/dbs/ ",
|
||||
}
|
||||
if err := normalizeBackupTargetRequest(&req); err != nil {
|
||||
t.Fatalf("normalize failed: %v", err)
|
||||
}
|
||||
if req.Endpoint != "garage.local:3900" {
|
||||
t.Fatalf("endpoint not normalized: %q", req.Endpoint)
|
||||
}
|
||||
if req.Bucket != "backups" || req.Prefix != "prod/dbs" {
|
||||
t.Fatalf("bucket/prefix not trimmed: %q %q", req.Bucket, req.Prefix)
|
||||
}
|
||||
|
||||
for i, bad := range []backupTargetRequest{
|
||||
{Endpoint: "e", Bucket: "b"}, // no name
|
||||
{Name: "n", Bucket: "b"}, // no endpoint
|
||||
{Name: "n", Endpoint: "e"}, // no bucket
|
||||
} {
|
||||
if err := normalizeBackupTargetRequest(&bad); err == nil {
|
||||
t.Fatalf("case %d: expected error", i)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestArchiveReaderUnwrapsTar(t *testing.T) {
|
||||
var buf bytes.Buffer
|
||||
tw := tar.NewWriter(&buf)
|
||||
payload := []byte("backup-bytes")
|
||||
if err := tw.WriteHeader(&tar.Header{Name: "b.tar.gz", Mode: 0o644, Size: int64(len(payload))}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := tw.Write(payload); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := tw.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
r := &archiveReader{tar: io.NopCloser(&buf)}
|
||||
got, err := io.ReadAll(r)
|
||||
if err != nil {
|
||||
t.Fatalf("read: %v", err)
|
||||
}
|
||||
if !bytes.Equal(got, payload) {
|
||||
t.Fatalf("payload mismatch: %q", got)
|
||||
}
|
||||
}
|
||||
@@ -54,7 +54,8 @@ type DatabaseService struct {
|
||||
NextBackupAt *time.Time `json:"next_backup_at,omitempty"`
|
||||
Provider string `json:"provider"` // managed | external
|
||||
PublicPort bool `json:"public_port" db:"public_port"`
|
||||
External *DatabaseExternalRef `json:"external,omitempty"` // connection target for external DBs
|
||||
BackupTargetID string `json:"backup_target_id,omitempty"` // S3-compatible archive destination
|
||||
External *DatabaseExternalRef `json:"external,omitempty"` // connection target for external DBs
|
||||
CreatedAt time.Time `json:"created_at" db:"created_at"`
|
||||
UpdatedAt time.Time `json:"updated_at" db:"updated_at"`
|
||||
ConnectionURL string `json:"connection_url"`
|
||||
@@ -106,6 +107,7 @@ type DatabaseBackup struct {
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
Size string `json:"size"`
|
||||
Status string `json:"status"` // completed, failed, in_progress
|
||||
Remote bool `json:"remote"` // archive also exists on the configured backup target
|
||||
}
|
||||
|
||||
// DatabaseSettings represents database configuration
|
||||
@@ -131,6 +133,7 @@ type DatabaseUpdateRequest struct {
|
||||
Plan string `json:"plan,omitempty"`
|
||||
BackupSchedule *string `json:"backup_schedule,omitempty"`
|
||||
PublicPort *bool `json:"public_port,omitempty"`
|
||||
BackupTargetID *string `json:"backup_target_id,omitempty"` // empty string clears
|
||||
}
|
||||
|
||||
// DatabaseActionRequest represents a request to perform database actions
|
||||
@@ -417,12 +420,12 @@ func (h *DatabaseHandler) UpdateDatabase(c *gin.Context) {
|
||||
req.Name = strings.TrimSpace(req.Name)
|
||||
req.Plan = strings.TrimSpace(req.Plan)
|
||||
|
||||
if req.Name == "" && req.Plan == "" && req.BackupSchedule == nil && req.PublicPort == nil {
|
||||
if req.Name == "" && req.Plan == "" && req.BackupSchedule == nil && req.PublicPort == nil && req.BackupTargetID == nil {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": "No fields to update"})
|
||||
return
|
||||
}
|
||||
|
||||
if req.PublicPort != nil {
|
||||
if req.PublicPort != nil || req.BackupTargetID != nil {
|
||||
row, err := h.queries.GetDatabaseServiceByIDAndUser(c.Request.Context(), sqlcdb.GetDatabaseServiceByIDAndUserParams{
|
||||
ID: databaseID,
|
||||
UserID: userID,
|
||||
@@ -431,23 +434,47 @@ func (h *DatabaseHandler) UpdateDatabase(c *gin.Context) {
|
||||
c.JSON(http.StatusNotFound, gin.H{"error": "Database not found", "code": "NOT_FOUND"})
|
||||
return
|
||||
}
|
||||
if row.Provider == "external" {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": "public_port applies to managed databases — external databases are already reachable", "code": "VALIDATION"})
|
||||
return
|
||||
if req.PublicPort != nil {
|
||||
if row.Provider == "external" {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": "public_port applies to managed databases — external databases are already reachable", "code": "VALIDATION"})
|
||||
return
|
||||
}
|
||||
if err := h.queries.SetDatabaseServicePublicPortByIDAndUser(c.Request.Context(), sqlcdb.SetDatabaseServicePublicPortByIDAndUserParams{
|
||||
PublicPort: *req.PublicPort,
|
||||
UpdatedAt: sql.NullTime{Time: time.Now(), Valid: true},
|
||||
ID: databaseID,
|
||||
UserID: userID,
|
||||
}); err != nil {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to update public port"})
|
||||
return
|
||||
}
|
||||
// Port bindings are baked into the container — recreate it so the
|
||||
// new bind scope takes effect. The data volume survives.
|
||||
if *req.PublicPort != row.PublicPort {
|
||||
go h.reprovisionManagedDatabase(databaseID, row.Name, row.Type)
|
||||
}
|
||||
}
|
||||
if err := h.queries.SetDatabaseServicePublicPortByIDAndUser(c.Request.Context(), sqlcdb.SetDatabaseServicePublicPortByIDAndUserParams{
|
||||
PublicPort: *req.PublicPort,
|
||||
UpdatedAt: sql.NullTime{Time: time.Now(), Valid: true},
|
||||
ID: databaseID,
|
||||
UserID: userID,
|
||||
}); err != nil {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to update public port"})
|
||||
return
|
||||
}
|
||||
// Port bindings are baked into the container — recreate it so the
|
||||
// new bind scope takes effect. The data volume survives.
|
||||
if *req.PublicPort != row.PublicPort {
|
||||
go h.reprovisionManagedDatabase(databaseID, row.Name, row.Type)
|
||||
if req.BackupTargetID != nil {
|
||||
if row.Provider == "external" {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": "backup targets apply to managed databases — external backups are not supported yet", "code": "VALIDATION"})
|
||||
return
|
||||
}
|
||||
targetID := strings.TrimSpace(*req.BackupTargetID)
|
||||
if targetID != "" {
|
||||
if _, err := h.queries.GetBackupTargetByIDAndUser(c.Request.Context(), sqlcdb.GetBackupTargetByIDAndUserParams{ID: targetID, UserID: userID}); err != nil {
|
||||
c.JSON(http.StatusNotFound, gin.H{"error": "Backup target not found", "code": "NOT_FOUND"})
|
||||
return
|
||||
}
|
||||
}
|
||||
if err := h.queries.SetDatabaseBackupTargetByIDAndUser(c.Request.Context(), sqlcdb.SetDatabaseBackupTargetByIDAndUserParams{
|
||||
BackupTargetID: sql.NullString{String: targetID, Valid: targetID != ""},
|
||||
UpdatedAt: sql.NullTime{Time: time.Now(), Valid: true},
|
||||
ID: databaseID,
|
||||
UserID: userID,
|
||||
}); err != nil {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to update backup target"})
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -991,6 +1018,7 @@ func (h *DatabaseHandler) resolveBackupConfig(ctx context.Context, userID string
|
||||
CreatedAt: createdAt,
|
||||
Size: row.Size,
|
||||
Status: row.Status,
|
||||
Remote: row.RemoteKey.Valid && row.RemoteKey.String != "",
|
||||
})
|
||||
if lastBackup == nil && row.Status == "completed" {
|
||||
t := createdAt
|
||||
@@ -1144,6 +1172,7 @@ func mapDatabaseServiceRow(row sqlcdb.DatabaseService) DatabaseService {
|
||||
NextBackupAt: databaseNullTimePtr(row.NextBackupAt),
|
||||
Provider: row.Provider,
|
||||
PublicPort: row.PublicPort,
|
||||
BackupTargetID: databaseNullString(row.BackupTargetID),
|
||||
ConnectionURL: databaseNullString(row.ConnectionUrl),
|
||||
CreatedAt: databaseNullTime(row.CreatedAt),
|
||||
UpdatedAt: databaseNullTime(row.UpdatedAt),
|
||||
@@ -1815,10 +1844,17 @@ func (h *DatabaseHandler) createBackupProcess(databaseID, backupID, backupPath s
|
||||
_ = h.setDatabaseBackupStatus(backupID, "failed", "0 B", false)
|
||||
return
|
||||
}
|
||||
if dbRow, err := h.queries.GetDatabaseServiceByID(ctx, databaseID); err == nil {
|
||||
if key, err := h.uploadBackupToTarget(ctx, dbRow, backupPath, backupID); err != nil {
|
||||
log.Printf("containr: backup %s remote upload failed: %v (local archive kept)", backupID, err)
|
||||
} else if key != "" {
|
||||
log.Printf("containr: backup %s shipped to %s", backupID, key)
|
||||
}
|
||||
}
|
||||
_ = h.setDatabaseBackupStatus(backupID, "completed", sizeLabel, true)
|
||||
}
|
||||
|
||||
func (h *DatabaseHandler) restoreBackupProcess(databaseID, _ string, backupPath string) {
|
||||
func (h *DatabaseHandler) restoreBackupProcess(databaseID, backupID string, backupPath string) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Minute)
|
||||
defer cancel()
|
||||
|
||||
@@ -1827,6 +1863,22 @@ func (h *DatabaseHandler) restoreBackupProcess(databaseID, _ string, backupPath
|
||||
return
|
||||
}
|
||||
|
||||
// Restore the archive from the configured S3 target when the local copy
|
||||
// is gone (fresh node, pruned volume) — local takes precedence.
|
||||
if dbRow, err := h.queries.GetDatabaseServiceByID(ctx, databaseID); err == nil && dbRow.BackupTargetID.Valid && dbRow.BackupTargetID.String != "" {
|
||||
if ok, _ := h.backupArchiveExists(ctx, backupPath); !ok {
|
||||
if b, err := h.queries.GetDatabaseBackupByIDAndDatabaseAndUser(ctx, sqlcdb.GetDatabaseBackupByIDAndDatabaseAndUserParams{
|
||||
ID: backupID, DatabaseID: databaseID, UserID: dbRow.UserID,
|
||||
}); err == nil && b.RemoteKey.Valid && b.RemoteKey.String != "" {
|
||||
if err := h.downloadBackupToVolume(ctx, dbRow, b.RemoteKey.String, backupPath); err != nil {
|
||||
log.Printf("containr: remote restore fetch failed for backup %s: %v", backupID, err)
|
||||
_ = h.setDatabaseStatus(databaseID, "error")
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
_ = h.setDatabaseStatus(databaseID, "building")
|
||||
_ = h.stopManagedDatabase(databaseID)
|
||||
|
||||
|
||||
@@ -354,6 +354,13 @@ func SetupRoutes(router *gin.Engine, db *database.DB, redis *database.Redis, cfg
|
||||
// One-off exec console (docker exec, 30s ceiling)
|
||||
authed.POST("/services/:id/exec", handleExecInService)
|
||||
|
||||
// S3-compatible backup targets — owner-scoped, keys encrypted at rest.
|
||||
authed.GET("/backup-targets", databaseHandler.ListBackupTargets)
|
||||
authed.POST("/backup-targets", databaseHandler.CreateBackupTarget)
|
||||
authed.PUT("/backup-targets/:id", databaseHandler.UpdateBackupTarget)
|
||||
authed.DELETE("/backup-targets/:id", databaseHandler.DeleteBackupTarget)
|
||||
authed.POST("/backup-targets/:id/test", databaseHandler.TestBackupTarget)
|
||||
|
||||
// Private-image registries — owner-scoped, credentials encrypted at rest.
|
||||
authed.GET("/registries", handleListRegistries)
|
||||
authed.POST("/registries", handleCreateRegistry)
|
||||
|
||||
Reference in new issue
Block a user