feat(ops): /operations endpoint + page — live jobs, queue depth, failures

GET /api/v1/operations aggregates active deployments, per-service
deploy-queue depth via Queue.Snapshot(), 24h deployment failures,
cron runs and backups scoped to the caller's projects (admins see all).

Frontend /operations page (Operate nav) shows in-flight count, queued
depth, failures and backup stats with live refetch while deploys are
active. CLI `containr operations` + MCP containr_operations_get.

Generated with [Devin](https://devin.ai)

Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com>

app/backend/internal/api/operations.go

@@ -0,0 +1,158 @@
+package api
+
+import (
+	"net/http"
+	"time"
+
+	"containr/internal/database"
+	"containr/internal/deployqueue"
+
+	"github.com/gin-gonic/gin"
+)
+
+// GET /operations — a single cross-cutting view of platform work: active
+// and recent deployments, the per-service deploy queue, recent cron runs,
+// and recent backups. Scoped to the caller's projects (admins see all).
+
+type operationDeployment struct {
+	ID          string     `json:"id"`
+	ServiceID   string     `json:"service_id"`
+	ServiceName string     `json:"service_name"`
+	ProjectName string     `json:"project_name"`
+	Status      string     `json:"status"`
+	Image       string     `json:"image"`
+	Error       *string    `json:"error,omitempty"`
+	StartedAt   *time.Time `json:"started_at,omitempty"`
+	CompletedAt *time.Time `json:"completed_at,omitempty"`
+	CreatedAt   time.Time  `json:"created_at"`
+}
+
+type operationCronRun struct {
+	ID         string     `json:"id"`
+	JobName    string     `json:"job_name"`
+	Schedule   string     `json:"schedule"`
+	Status     string     `json:"status"`
+	StartedAt  time.Time  `json:"started_at"`
+	FinishedAt *time.Time `json:"finished_at,omitempty"`
+	Error      *string    `json:"error,omitempty"`
+}
+
+type operationBackup struct {
+	ID         string     `json:"id"`
+	DatabaseID string     `json:"database_id"`
+	DBName     string     `json:"database_name"`
+	Status     string     `json:"status"`
+	Size       string     `json:"size"`
+	CreatedAt  time.Time  `json:"created_at"`
+	Completed  *time.Time `json:"completed_at,omitempty"`
+}
+
+func handleGetOperations(c *gin.Context) {
+	userID, ok := requireAuthenticatedUserID(c)
+	if !ok {
+		return
+	}
+	db := c.MustGet("db").(*database.DB)
+	isAdmin := contextIsAdmin(c)
+
+	// Same visibility rule as the deployment list endpoint.
+	visibility := `p.is_approved OR p.owner_id = $1 OR $2::bool
+	    OR EXISTS (SELECT 1 FROM project_members pm WHERE pm.project_id = p.id AND pm.user_id = $1)`
+
+	active := []operationDeployment{}
+	rows, err := db.Query(
+		`SELECT d.id, d.service_id, s.name, p.name, d.status,
+		        COALESCE(d.image_name, ''), d.error, d.started_at, d.completed_at, d.created_at
+		 FROM deployments d
+		 JOIN services s ON s.id = d.service_id
+		 JOIN projects p ON p.id = s.project_id
+		 WHERE d.status IN ('queued','pending','building','deploying','rolling_back')
+		   AND (`+visibility+`)
+		 ORDER BY d.created_at ASC`, userID, isAdmin)
+	if err == nil {
+		defer rows.Close()
+		for rows.Next() {
+			var d operationDeployment
+			var sid string
+			_ = rows.Scan(&d.ID, &sid, &d.ServiceName, &d.ProjectName, &d.Status, &d.Image, &d.Error, &d.StartedAt, &d.CompletedAt, &d.CreatedAt)
+			d.ServiceID = sid
+			active = append(active, d)
+		}
+	}
+
+	recentFailed := []operationDeployment{}
+	rows2, err2 := db.Query(
+		`SELECT d.id, d.service_id, s.name, p.name, d.status,
+		        COALESCE(d.image_name, ''), d.error, d.started_at, d.completed_at, d.created_at
+		 FROM deployments d
+		 JOIN services s ON s.id = d.service_id
+		 JOIN projects p ON p.id = s.project_id
+		 WHERE d.status IN ('failed','cancelled','rolled_back')
+		   AND d.updated_at > NOW() - INTERVAL '24 hours'
+		   AND (`+visibility+`)
+		 ORDER BY d.updated_at DESC
+		 LIMIT 20`, userID, isAdmin)
+	if err2 == nil {
+		defer rows2.Close()
+		for rows2.Next() {
+			var d operationDeployment
+			var sid string
+			_ = rows2.Scan(&d.ID, &sid, &d.ServiceName, &d.ProjectName, &d.Status, &d.Image, &d.Error, &d.StartedAt, &d.CompletedAt, &d.CreatedAt)
+			d.ServiceID = sid
+			recentFailed = append(recentFailed, d)
+		}
+	}
+
+	cronRuns := []operationCronRun{}
+	rows3, err3 := db.Query(
+		`SELECT e.id, j.name, j.schedule, e.status, e.started_at, e.finished_at, e.error
+		 FROM cron_executions e
+		 JOIN cron_jobs j ON j.id = e.cron_job_id
+		 JOIN projects p ON p.id = j.project_id
+		 WHERE e.started_at > NOW() - INTERVAL '24 hours'
+		   AND (`+visibility+`)
+		 ORDER BY e.started_at DESC
+		 LIMIT 30`, userID, isAdmin)
+	if err3 == nil {
+		defer rows3.Close()
+		for rows3.Next() {
+			var r operationCronRun
+			_ = rows3.Scan(&r.ID, &r.JobName, &r.Schedule, &r.Status, &r.StartedAt, &r.FinishedAt, &r.Error)
+			cronRuns = append(cronRuns, r)
+		}
+	}
+
+	backups := []operationBackup{}
+	rows4, err4 := db.Query(
+		`SELECT b.id, b.database_id, ds.name, b.status, b.size, b.created_at, b.completed_at
+		 FROM database_backups b
+		 JOIN database_services ds ON ds.id = b.database_id
+		 WHERE b.created_at > NOW() - INTERVAL '24 hours'
+		   AND (ds.user_id = $1 OR $2::bool)
+		 ORDER BY b.created_at DESC
+		 LIMIT 20`, userID, isAdmin)
+	if err4 == nil {
+		defer rows4.Close()
+		for rows4.Next() {
+			var b operationBackup
+			_ = rows4.Scan(&b.ID, &b.DatabaseID, &b.DBName, &b.Status, &b.Size, &b.CreatedAt, &b.Completed)
+			backups = append(backups, b)
+		}
+	}
+
+	var queue []deployqueue.ServiceQueueState
+	if queueValue, exists := c.Get("deploy_queue"); exists && queueValue != nil {
+		queue = queueValue.(*deployqueue.Queue).Snapshot()
+	}
+	if queue == nil {
+		queue = []deployqueue.ServiceQueueState{}
+	}
+
+	c.JSON(http.StatusOK, gin.H{
+		"active_deployments": active,
+		"recent_failures":    recentFailed,
+		"cron_runs":          cronRuns,
+		"backups":            backups,
+		"deploy_queue":       queue,
+	})
+}

app/backend/internal/api/routes.go

@@ -333,6 +333,9 @@ func SetupRoutes(router *gin.Engine, db *database.DB, redis *database.Redis, cfg
 			// WebSocket endpoint
 			authed.GET("/ws", handleWebSocket)
 
+			// Cross-cutting ops view — active/queued/failed work
+			authed.GET("/operations", handleGetOperations)
+
 			// Outbound webhooks — signed event delivery to user HTTP endpoints
 			authed.GET("/webhooks", handleListOutboundWebhooks)
 			authed.POST("/webhooks", handleCreateOutboundWebhook)

app/backend/internal/cli/commands/operations.go

@@ -0,0 +1,24 @@
+package commands
+
+import (
+	"github.com/spf13/cobra"
+)
+
+// OperationsCmd shows the cross-cutting operations view.
+var OperationsCmd = &cobra.Command{
+	Use:     "operations",
+	Aliases: []string{"ops"},
+	Short:   "Show active and recent platform work (deploys, cron, backups)",
+	RunE: func(cmd *cobra.Command, args []string) error {
+		c, err := client()
+		if err != nil {
+			return err
+		}
+		data, err := c.Do("GET", "/operations", nil)
+		if err != nil {
+			return err
+		}
+		PrintRaw(data)
+		return nil
+	},
+}

app/backend/internal/cli/root.go

@@ -88,6 +88,7 @@ func init() {
 	rootCmd.AddCommand(commands.AdminCmd)
 	rootCmd.AddCommand(commands.NotificationsCmd)
 	rootCmd.AddCommand(commands.WebhooksCmd)
+	rootCmd.AddCommand(commands.OperationsCmd)
 	rootCmd.AddCommand(commands.UpCmd)
 }
 

app/backend/internal/deployqueue/queue.go

@@ -129,6 +129,27 @@ func (q *Queue) Cancel(serviceID, deploymentID uuid.UUID) CancelState {
 	return NotFound
 }
 
+// ServiceQueueState reports queue depth for one service.
+type ServiceQueueState struct {
+	ServiceID uuid.UUID `json:"service_id"`
+	Running   bool      `json:"running"`
+	Queued    int       `json:"queued"`
+}
+
+// Snapshot returns per-service queue depth for observability.
+func (q *Queue) Snapshot() []ServiceQueueState {
+	q.mu.Lock()
+	defer q.mu.Unlock()
+	out := make([]ServiceQueueState, 0, len(q.states))
+	for id, st := range q.states {
+		if !st.running && len(st.pending) == 0 {
+			continue
+		}
+		out = append(out, ServiceQueueState{ServiceID: id, Running: st.running, Queued: len(st.pending)})
+	}
+	return out
+}
+
 // Do runs fn through the queue and waits for completion — for synchronous
 // endpoints (redeploy) that still must not overlap an in-flight deployment.
 // ctx bounds the wait, not the job; on timeout the job still runs to

app/backend/internal/deployqueue/queue_test.go

@@ -125,6 +125,40 @@ func TestCancelActiveCancelsContext(t *testing.T) {
 	}
 }
 
+func TestSnapshot(t *testing.T) {
+	q := New()
+	if got := q.Snapshot(); len(got) != 0 {
+		t.Fatalf("empty queue snapshot = %v, want empty", got)
+	}
+
+	svc := uuid.New()
+	release := make(chan struct{})
+	q.Enqueue(svc, Job{Run: func(ctx context.Context) { <-release }})
+	q.Enqueue(svc, Job{Run: func(ctx context.Context) {}})
+	q.Enqueue(svc, Job{Run: func(ctx context.Context) {}})
+
+	deadline := time.After(2 * time.Second)
+	for {
+		snap := q.Snapshot()
+		if len(snap) == 1 && snap[0].Running {
+			if snap[0].ServiceID != svc {
+				t.Fatalf("snapshot service = %v, want %v", snap[0].ServiceID, svc)
+			}
+			if snap[0].Queued != 2 {
+				t.Fatalf("queued = %d, want 2", snap[0].Queued)
+			}
+			close(release)
+			return
+		}
+		select {
+		case <-deadline:
+			t.Fatalf("snapshot never showed running job: %v", snap)
+		default:
+			time.Sleep(time.Millisecond)
+		}
+	}
+}
+
 func TestEnqueuePosition(t *testing.T) {
 	q := New()
 	svc := uuid.New()

app/backend/internal/mcpserver/server.go

@@ -147,6 +147,8 @@ func init() {
 	register(spec{"containr_webhooks_deliveries", "List recent delivery attempts for a webhook. Args: id.", "GET", "/webhooks/{id}/deliveries", []string{"id"}, nil, false})
 	register(spec{"containr_webhooks_test", "Queue a test ping delivery for a webhook. Args: id.", "POST", "/webhooks/{id}/test", []string{"id"}, nil, false})
 
+	register(spec{"containr_operations_get", "Cross-cutting operations view: active deployments, deploy queue depth, recent failures, cron runs, backups.", "GET", "/operations", nil, nil, false})
+
 	// Nodes / agent tokens (admin)
 	register(spec{"containr_nodes_list", "List node agents (admin)", "GET", "/agents", nil, nil, false})
 	register(spec{"containr_nodes_get", "Get a node agent (admin)", "GET", "/agents/{id}", []string{"id"}, nil, false})