From 302c595380f25d61540a7e319bc2de477e106085 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Dvo=C5=99=C3=A1k?= <150935816+Dvorinka@users.noreply.github.com> Date: Sat, 10 Oct 2026 03:14:03 +0200 Subject: [PATCH] refactor(db): port metrics storage + service domains to sqlc (#75) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit First tranche of the ~300-site raw SQL port. Metrics: node/service/ instance rows via generated queries; aggregation moved off TimescaleDB's time_bucket onto portable date_bin (PG14+); interval uptime returns epoch seconds; per-container writes dropped (container_metrics FKs a different model — collector shape never matched). Domains: all reads/writes through sqlcdb including the create transaction via WithTx; default-domain and services.domain invariants unchanged. Both paths covered by live Postgres tests gated on METRICS_TEST_DSN. Generated with [Devin](https://devin.ai) Co-authored-by: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- ROADMAP2.md | 2 +- app/backend/internal/api/domains.go | 115 ++-- app/backend/internal/api/domains_test.go | 121 +++- .../internal/database/sqlcdb/domains.sql.go | 242 +++++++ .../internal/database/sqlcdb/metrics.sql.go | 623 ++++++++++++++++++ .../internal/database/sqlcdb/models.go | 84 +++ .../internal/database/sqlcdb/querier.go | 22 + app/backend/internal/metrics/storage.go | 591 ++++++++--------- app/backend/internal/metrics/storage_test.go | 130 ++++ app/backend/sqlc/queries/domains.sql | 46 ++ app/backend/sqlc/queries/metrics.sql | 147 +++++ app/backend/sqlc/schema.sql | 87 +++ 12 files changed, 1825 insertions(+), 385 deletions(-) create mode 100644 app/backend/internal/database/sqlcdb/domains.sql.go create mode 100644 app/backend/internal/database/sqlcdb/metrics.sql.go create mode 100644 app/backend/internal/metrics/storage_test.go create mode 100644 app/backend/sqlc/queries/domains.sql create mode 100644 app/backend/sqlc/queries/metrics.sql diff --git a/ROADMAP2.md b/ROADMAP2.md index c21b2a6..43b4e00 100644 --- a/ROADMAP2.md +++ b/ROADMAP2.md @@ -537,7 +537,7 @@ and wired refs, one click or one CLI call. workspace, CLI `environments` + MCP parity. Per-env vars/domains ride the service's own scope (one env per service); promote ships via preview promote. -- [ ] **sqlc port** — ~140 raw `database/sql` sites → `sqlc/queries/`. +- [ ] **sqlc port** — ~300 raw `database/sql` sites → `sqlc/queries/`. Done in tranches (dynamic queries stay raw): metrics storage + service domains ported. - [ ] **Tests** — canvas/wizard frontend coverage, deployment/template/ webhook table tests, Playwright e2e golden path. - [x] **First-run wizard** — public `GET /setup/status` + authed diff --git a/app/backend/internal/api/domains.go b/app/backend/internal/api/domains.go index 0834c1c..46ba044 100644 --- a/app/backend/internal/api/domains.go +++ b/app/backend/internal/api/domains.go @@ -44,19 +44,26 @@ func validHostname(d string) bool { } func loadServiceDomains(db *database.DB, serviceID uuid.UUID) []ServiceDomain { - rows, err := db.Query( - `SELECT id, service_id, domain, is_default, cert_type, cert_status, last_checked_at, created_at - FROM service_domains WHERE service_id = $1 ORDER BY is_default DESC, created_at ASC`, serviceID) + rows, err := sqlcdb.New(db.DB).ListServiceDomains(context.Background(), serviceID) if err != nil { return nil } - defer rows.Close() - out := []ServiceDomain{} - for rows.Next() { - var d ServiceDomain - if err := rows.Scan(&d.ID, &d.ServiceID, &d.Domain, &d.IsDefault, &d.CertType, &d.CertStatus, &d.LastCheckedAt, &d.CreatedAt); err == nil { - out = append(out, d) + out := make([]ServiceDomain, 0, len(rows)) + for _, r := range rows { + d := ServiceDomain{ + ID: r.ID, + ServiceID: r.ServiceID, + Domain: r.Domain, + IsDefault: r.IsDefault, + CertType: r.CertType, + CertStatus: r.CertStatus, + CreatedAt: r.CreatedAt, } + if r.LastCheckedAt.Valid { + t := r.LastCheckedAt.Time + d.LastCheckedAt = &t + } + out = append(out, d) } return out } @@ -104,22 +111,20 @@ func dnsLabel(name string) string { // base domain; spread services union every eligible node's base domain. // Local services get nothing — there is no instance-level base domain yet. func serviceAutoDomains(db *database.DB, serviceID uuid.UUID) []string { - var name string - var nodeID sql.NullString - var spread bool - if err := db.QueryRow( - `SELECT name, node_id, COALESCE(spread, false) FROM services WHERE id = $1`, serviceID, - ).Scan(&name, &nodeID, &spread); err != nil { + ctx := context.Background() + q := sqlcdb.New(db.DB) + brief, err := q.GetServicePlacementBrief(ctx, serviceID) + if err != nil { return nil } - label := dnsLabel(name) + label := dnsLabel(brief.Name) if label == "" { return nil } var bases []string - ctx := context.Background() - q := sqlcdb.New(db.DB) + nodeID := brief.NodeID + spread := brief.Spread switch { case nodeID.Valid && nodeID.String != "": if a, err := q.GetAgent(ctx, nodeID.String); err == nil && a.DefaultDomain != "" { @@ -142,24 +147,25 @@ func serviceAutoDomains(db *database.DB, serviceID uuid.UUID) []string { // syncDefaultDomain writes the is_default row back to services.domain so // legacy reads (CLI tables, public_url) keep working. func syncDefaultDomain(db *database.DB, serviceID uuid.UUID) { - var domain string - err := db.QueryRow( - `SELECT domain FROM service_domains WHERE service_id = $1 AND is_default`, serviceID).Scan(&domain) + ctx := context.Background() + q := sqlcdb.New(db.DB) + domain, err := q.GetServiceDomainDefault(ctx, serviceID) if err == sql.ErrNoRows { - err = db.QueryRow( - `SELECT domain FROM service_domains WHERE service_id = $1 ORDER BY created_at ASC LIMIT 1`, serviceID).Scan(&domain) + domain, err = q.GetServiceDomainFirst(ctx, serviceID) } if err == sql.ErrNoRows { domain = "" } - _, _ = db.Exec(`UPDATE services SET domain = $1 WHERE id = $2`, domain, serviceID) + _ = q.SetServiceLegacyDomain(ctx, sqlcdb.SetServiceLegacyDomainParams{Domain: domain, ID: serviceID}) } // serviceAccess loads the maintenance/basic-auth columns for the runtime spec. func serviceAccess(db *database.DB, serviceID uuid.UUID) (maintenance bool, basicAuth string) { - _ = db.QueryRow(`SELECT maintenance_mode, basic_auth_users FROM services WHERE id = $1`, serviceID). - Scan(&maintenance, &basicAuth) - return maintenance, basicAuth + row, err := sqlcdb.New(db.DB).GetServiceAccess(context.Background(), serviceID) + if err != nil { + return false, "" + } + return row.MaintenanceMode, row.BasicAuthUsers } // maintenanceURL is the redirect target for maintenance mode — the public @@ -204,11 +210,12 @@ func basicAuthUsernames(encoded string) []string { // serviceWriteAccess mirrors the service update handler's access check. func serviceWriteAccess(c *gin.Context, db *database.DB, serviceID uuid.UUID) bool { userID, _ := c.Get("user_id") - var ok bool - err := db.QueryRow( - `SELECT EXISTS(SELECT 1 FROM services s JOIN projects p ON s.project_id = p.id - WHERE s.id = $1 AND (p.owner_id = $2 OR $3::bool))`, - serviceID, userID, contextIsAdmin(c)).Scan(&ok) + uid, _ := userID.(uuid.UUID) + ok, err := sqlcdb.New(db.DB).ServiceWriteAccess(context.Background(), sqlcdb.ServiceWriteAccessParams{ + ID: serviceID, + OwnerID: uid, + Column3: contextIsAdmin(c), + }) return err == nil && ok } @@ -255,9 +262,11 @@ func handleAddServiceDomain(c *gin.Context) { return } + ctx := c.Request.Context() + q := sqlcdb.New(db.(*database.DB).DB) + // First domain on a service becomes the default automatically. - var count int - _ = db.(*database.DB).QueryRow(`SELECT COUNT(*) FROM service_domains WHERE service_id = $1`, serviceID).Scan(&count) + count, _ := q.CountServiceDomains(ctx, serviceID) makeDefault := req.IsDefault || count == 0 tx, err := db.(*database.DB).Begin() @@ -266,19 +275,18 @@ func handleAddServiceDomain(c *gin.Context) { return } defer tx.Rollback() + qtx := q.WithTx(tx) if makeDefault { - if _, err := tx.Exec(`UPDATE service_domains SET is_default = false WHERE service_id = $1`, serviceID); err != nil { + if err := qtx.ClearServiceDomainDefault(ctx, serviceID); err != nil { respondError(c, http.StatusInternalServerError, "INTERNAL", "Failed to add domain") return } } - var inserted ServiceDomain - err = tx.QueryRow( - `INSERT INTO service_domains (service_id, domain, is_default) - VALUES ($1, $2, $3) - RETURNING id, service_id, domain, is_default, cert_type, cert_status, created_at`, - serviceID, domain, makeDefault). - Scan(&inserted.ID, &inserted.ServiceID, &inserted.Domain, &inserted.IsDefault, &inserted.CertType, &inserted.CertStatus, &inserted.CreatedAt) + row, err := qtx.CreateServiceDomain(ctx, sqlcdb.CreateServiceDomainParams{ + ServiceID: serviceID, + Domain: domain, + IsDefault: makeDefault, + }) if err != nil { if strings.Contains(err.Error(), "duplicate") || strings.Contains(err.Error(), "unique") { respondError(c, http.StatusConflict, "CONFLICT", "Domain already attached to this service") @@ -292,7 +300,15 @@ func handleAddServiceDomain(c *gin.Context) { return } syncDefaultDomain(db.(*database.DB), serviceID) - c.JSON(http.StatusCreated, gin.H{"domain": inserted}) + c.JSON(http.StatusCreated, gin.H{"domain": ServiceDomain{ + ID: row.ID, + ServiceID: row.ServiceID, + Domain: row.Domain, + IsDefault: row.IsDefault, + CertType: row.CertType, + CertStatus: row.CertStatus, + CreatedAt: row.CreatedAt, + }}) } func handleDeleteServiceDomain(c *gin.Context) { @@ -315,9 +331,8 @@ func handleDeleteServiceDomain(c *gin.Context) { respondError(c, http.StatusBadRequest, "VALIDATION", "Invalid domain ID") return } - res, err := db.(*database.DB).Exec( - `DELETE FROM service_domains WHERE id = $1 AND service_id = $2`, domainID, serviceID) - affected, _ := res.RowsAffected() + affected, err := sqlcdb.New(db.(*database.DB).DB).DeleteServiceDomain( + c.Request.Context(), sqlcdb.DeleteServiceDomainParams{ID: domainID, ServiceID: serviceID}) if err != nil || affected == 0 { respondError(c, http.StatusNotFound, "NOT_FOUND", "Domain not found") return @@ -346,14 +361,12 @@ func handleSetDefaultDomain(c *gin.Context) { respondError(c, http.StatusBadRequest, "VALIDATION", "Invalid domain ID") return } - res, err := db.(*database.DB).Exec( - `UPDATE service_domains SET is_default = (id = $1) WHERE service_id = $2`, - domainID, serviceID) + affected, err := sqlcdb.New(db.(*database.DB).DB).SetServiceDomainDefault( + c.Request.Context(), sqlcdb.SetServiceDomainDefaultParams{ID: domainID, ServiceID: serviceID}) if err != nil { respondError(c, http.StatusInternalServerError, "INTERNAL", "Failed to set default domain") return } - affected, _ := res.RowsAffected() if affected == 0 { respondError(c, http.StatusNotFound, "NOT_FOUND", "Domain not found") return @@ -421,7 +434,7 @@ func handleCheckServiceDomains(c *gin.Context) { r.Status = "ok" } results = append(results, r) - _, _ = dbh.Exec(`UPDATE service_domains SET last_checked_at = NOW() WHERE id = $1`, d.ID) + _ = sqlcdb.New(dbh.DB).TouchServiceDomainChecked(c.Request.Context(), d.ID) } c.JSON(http.StatusOK, gin.H{"domains": results}) } diff --git a/app/backend/internal/api/domains_test.go b/app/backend/internal/api/domains_test.go index b065d2a..305e10e 100644 --- a/app/backend/internal/api/domains_test.go +++ b/app/backend/internal/api/domains_test.go @@ -1,6 +1,28 @@ package api -import "testing" +import ( + "context" + "os" + "strings" + "testing" + + "containr/internal/database" + "containr/internal/database/sqlcdb" + + "github.com/google/uuid" +) + +func testDSNString(t *testing.T) string { + t.Helper() + dsn := os.Getenv("METRICS_TEST_DSN") + if dsn == "" { + dsn = os.Getenv("DATABASE_URL") + } + if dsn == "" { + t.Skip("METRICS_TEST_DSN not set") + } + return dsn +} func TestDNSLabel(t *testing.T) { cases := map[string]string{ @@ -24,3 +46,100 @@ func TestDNSLabelLengthCap(t *testing.T) { t.Fatalf("label over 63 chars: %d", len(got)) } } + +// Live invariant test against real Postgres — set METRICS_TEST_DSN (or +// DATABASE_URL). Verifies the generated domain queries preserve the +// default-domain and legacy services.domain invariants. +func TestServiceDomainsLiveInvariants(t *testing.T) { + dsn := testDSNString(t) + db, err := database.NewConnection(dsn) + if err != nil { + t.Fatalf("connect: %v", err) + } + defer db.Close() + ctx := context.Background() + q := sqlcdb.New(db.DB) + + userID := uuid.New() + projectID := uuid.New() + envID := uuid.New() + serviceID := uuid.New() + suffix := strings.ReplaceAll(serviceID.String()[:8], "-", "") + + must := func(err error) { + t.Helper() + if err != nil { + t.Fatalf("setup: %v", err) + } + } + _, err = db.Exec(`INSERT INTO users (id, email, password_hash, name) VALUES ($1, $2, 'x', 'test')`, + userID, "domtest-"+suffix+"@test.local") + must(err) + _, err = db.Exec(`INSERT INTO projects (id, name, owner_id, is_approved) VALUES ($1, $2, $3, true)`, + projectID, "domtest-"+suffix, userID) + must(err) + _, err = db.Exec(`INSERT INTO environments (id, name, project_id) VALUES ($1, 'production', $2)`, + envID, projectID) + must(err) + _, err = db.Exec(`INSERT INTO services (id, name, project_id, environment_id, service_type, source_type) + VALUES ($1, 'domtest-`+suffix+`', $2, $3, 'web', 'image')`, + serviceID, projectID, envID) + must(err) + defer func() { + _, _ = db.Exec(`DELETE FROM users WHERE id = $1`, userID) + }() + + d1, err := q.CreateServiceDomain(ctx, sqlcdb.CreateServiceDomainParams{ + ServiceID: serviceID, Domain: "a-" + suffix + ".example.com", IsDefault: true}) + if err != nil { + t.Fatalf("create d1: %v", err) + } + d2, err := q.CreateServiceDomain(ctx, sqlcdb.CreateServiceDomainParams{ + ServiceID: serviceID, Domain: "b-" + suffix + ".example.com", IsDefault: false}) + if err != nil { + t.Fatalf("create d2: %v", err) + } + + syncDefaultDomain(db, serviceID) + var legacy string + if err := db.QueryRow(`SELECT domain FROM services WHERE id = $1`, serviceID).Scan(&legacy); err != nil { + t.Fatalf("read legacy: %v", err) + } + if legacy != d1.Domain { + t.Fatalf("legacy domain = %q, want %q", legacy, d1.Domain) + } + + // Switch default: exactly one row stays default, legacy follows. + if err := q.ClearServiceDomainDefault(ctx, serviceID); err != nil { + t.Fatalf("clear: %v", err) + } + if _, err := q.SetServiceDomainDefault(ctx, sqlcdb.SetServiceDomainDefaultParams{ + ID: d2.ID, ServiceID: serviceID}); err != nil { + t.Fatalf("set default: %v", err) + } + syncDefaultDomain(db, serviceID) + if err := db.QueryRow(`SELECT domain FROM services WHERE id = $1`, serviceID).Scan(&legacy); err != nil { + t.Fatalf("read legacy2: %v", err) + } + if legacy != d2.Domain { + t.Fatalf("legacy after switch = %q, want %q", legacy, d2.Domain) + } + + doms := loadServiceDomains(db, serviceID) + if len(doms) != 2 || !doms[0].IsDefault || doms[0].Domain != d2.Domain { + t.Fatalf("list order wrong: %+v", doms) + } + + // Deleting the default falls back to the remaining domain. + if _, err := q.DeleteServiceDomain(ctx, sqlcdb.DeleteServiceDomainParams{ + ID: d2.ID, ServiceID: serviceID}); err != nil { + t.Fatalf("delete d2: %v", err) + } + syncDefaultDomain(db, serviceID) + if err := db.QueryRow(`SELECT domain FROM services WHERE id = $1`, serviceID).Scan(&legacy); err != nil { + t.Fatalf("read legacy3: %v", err) + } + if legacy != d1.Domain { + t.Fatalf("legacy after delete = %q, want %q", legacy, d1.Domain) + } +} diff --git a/app/backend/internal/database/sqlcdb/domains.sql.go b/app/backend/internal/database/sqlcdb/domains.sql.go new file mode 100644 index 0000000..8240034 --- /dev/null +++ b/app/backend/internal/database/sqlcdb/domains.sql.go @@ -0,0 +1,242 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.31.1 +// source: domains.sql + +package sqlcdb + +import ( + "context" + "database/sql" + "time" + + "github.com/google/uuid" +) + +const clearServiceDomainDefault = `-- name: ClearServiceDomainDefault :exec +UPDATE service_domains SET is_default = false WHERE service_id = $1 +` + +func (q *Queries) ClearServiceDomainDefault(ctx context.Context, serviceID uuid.UUID) error { + _, err := q.db.ExecContext(ctx, clearServiceDomainDefault, serviceID) + return err +} + +const countServiceDomains = `-- name: CountServiceDomains :one +SELECT COUNT(*) FROM service_domains WHERE service_id = $1 +` + +func (q *Queries) CountServiceDomains(ctx context.Context, serviceID uuid.UUID) (int64, error) { + row := q.db.QueryRowContext(ctx, countServiceDomains, serviceID) + var count int64 + err := row.Scan(&count) + return count, err +} + +const createServiceDomain = `-- name: CreateServiceDomain :one +INSERT INTO service_domains (service_id, domain, is_default) +VALUES ($1, $2, $3) +RETURNING id, service_id, domain, is_default, cert_type, cert_status, created_at +` + +type CreateServiceDomainParams struct { + ServiceID uuid.UUID `json:"service_id"` + Domain string `json:"domain"` + IsDefault bool `json:"is_default"` +} + +type CreateServiceDomainRow struct { + ID uuid.UUID `json:"id"` + ServiceID uuid.UUID `json:"service_id"` + Domain string `json:"domain"` + IsDefault bool `json:"is_default"` + CertType string `json:"cert_type"` + CertStatus string `json:"cert_status"` + CreatedAt time.Time `json:"created_at"` +} + +func (q *Queries) CreateServiceDomain(ctx context.Context, arg CreateServiceDomainParams) (CreateServiceDomainRow, error) { + row := q.db.QueryRowContext(ctx, createServiceDomain, arg.ServiceID, arg.Domain, arg.IsDefault) + var i CreateServiceDomainRow + err := row.Scan( + &i.ID, + &i.ServiceID, + &i.Domain, + &i.IsDefault, + &i.CertType, + &i.CertStatus, + &i.CreatedAt, + ) + return i, err +} + +const deleteServiceDomain = `-- name: DeleteServiceDomain :execrows +DELETE FROM service_domains WHERE id = $1 AND service_id = $2 +` + +type DeleteServiceDomainParams struct { + ID uuid.UUID `json:"id"` + ServiceID uuid.UUID `json:"service_id"` +} + +func (q *Queries) DeleteServiceDomain(ctx context.Context, arg DeleteServiceDomainParams) (int64, error) { + result, err := q.db.ExecContext(ctx, deleteServiceDomain, arg.ID, arg.ServiceID) + if err != nil { + return 0, err + } + return result.RowsAffected() +} + +const getServiceAccess = `-- name: GetServiceAccess :one +SELECT maintenance_mode, basic_auth_users FROM services WHERE id = $1 +` + +type GetServiceAccessRow struct { + MaintenanceMode bool `json:"maintenance_mode"` + BasicAuthUsers string `json:"basic_auth_users"` +} + +func (q *Queries) GetServiceAccess(ctx context.Context, id uuid.UUID) (GetServiceAccessRow, error) { + row := q.db.QueryRowContext(ctx, getServiceAccess, id) + var i GetServiceAccessRow + err := row.Scan(&i.MaintenanceMode, &i.BasicAuthUsers) + return i, err +} + +const getServiceDomainDefault = `-- name: GetServiceDomainDefault :one +SELECT domain FROM service_domains WHERE service_id = $1 AND is_default +` + +func (q *Queries) GetServiceDomainDefault(ctx context.Context, serviceID uuid.UUID) (string, error) { + row := q.db.QueryRowContext(ctx, getServiceDomainDefault, serviceID) + var domain string + err := row.Scan(&domain) + return domain, err +} + +const getServiceDomainFirst = `-- name: GetServiceDomainFirst :one +SELECT domain FROM service_domains WHERE service_id = $1 ORDER BY created_at ASC LIMIT 1 +` + +func (q *Queries) GetServiceDomainFirst(ctx context.Context, serviceID uuid.UUID) (string, error) { + row := q.db.QueryRowContext(ctx, getServiceDomainFirst, serviceID) + var domain string + err := row.Scan(&domain) + return domain, err +} + +const getServicePlacementBrief = `-- name: GetServicePlacementBrief :one +SELECT name, node_id, COALESCE(spread, false) AS spread FROM services WHERE id = $1 +` + +type GetServicePlacementBriefRow struct { + Name string `json:"name"` + NodeID sql.NullString `json:"node_id"` + Spread bool `json:"spread"` +} + +func (q *Queries) GetServicePlacementBrief(ctx context.Context, id uuid.UUID) (GetServicePlacementBriefRow, error) { + row := q.db.QueryRowContext(ctx, getServicePlacementBrief, id) + var i GetServicePlacementBriefRow + err := row.Scan(&i.Name, &i.NodeID, &i.Spread) + return i, err +} + +const listServiceDomains = `-- name: ListServiceDomains :many +SELECT id, service_id, domain, is_default, cert_type, cert_status, last_checked_at, created_at +FROM service_domains +WHERE service_id = $1 +ORDER BY is_default DESC, created_at ASC +` + +func (q *Queries) ListServiceDomains(ctx context.Context, serviceID uuid.UUID) ([]ServiceDomain, error) { + rows, err := q.db.QueryContext(ctx, listServiceDomains, serviceID) + if err != nil { + return nil, err + } + defer rows.Close() + items := []ServiceDomain{} + for rows.Next() { + var i ServiceDomain + if err := rows.Scan( + &i.ID, + &i.ServiceID, + &i.Domain, + &i.IsDefault, + &i.CertType, + &i.CertStatus, + &i.LastCheckedAt, + &i.CreatedAt, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const serviceWriteAccess = `-- name: ServiceWriteAccess :one +SELECT EXISTS( + SELECT 1 FROM services s JOIN projects p ON s.project_id = p.id + WHERE s.id = $1 AND (p.owner_id = $2 OR $3::bool) +) AS allowed +` + +type ServiceWriteAccessParams struct { + ID uuid.UUID `json:"id"` + OwnerID uuid.UUID `json:"owner_id"` + Column3 bool `json:"column_3"` +} + +func (q *Queries) ServiceWriteAccess(ctx context.Context, arg ServiceWriteAccessParams) (bool, error) { + row := q.db.QueryRowContext(ctx, serviceWriteAccess, arg.ID, arg.OwnerID, arg.Column3) + var allowed bool + err := row.Scan(&allowed) + return allowed, err +} + +const setServiceDomainDefault = `-- name: SetServiceDomainDefault :execrows +UPDATE service_domains SET is_default = (id = $1) WHERE service_id = $2 +` + +type SetServiceDomainDefaultParams struct { + ID uuid.UUID `json:"id"` + ServiceID uuid.UUID `json:"service_id"` +} + +func (q *Queries) SetServiceDomainDefault(ctx context.Context, arg SetServiceDomainDefaultParams) (int64, error) { + result, err := q.db.ExecContext(ctx, setServiceDomainDefault, arg.ID, arg.ServiceID) + if err != nil { + return 0, err + } + return result.RowsAffected() +} + +const setServiceLegacyDomain = `-- name: SetServiceLegacyDomain :exec +UPDATE services SET domain = $1 WHERE id = $2 +` + +type SetServiceLegacyDomainParams struct { + Domain string `json:"domain"` + ID uuid.UUID `json:"id"` +} + +func (q *Queries) SetServiceLegacyDomain(ctx context.Context, arg SetServiceLegacyDomainParams) error { + _, err := q.db.ExecContext(ctx, setServiceLegacyDomain, arg.Domain, arg.ID) + return err +} + +const touchServiceDomainChecked = `-- name: TouchServiceDomainChecked :exec +UPDATE service_domains SET last_checked_at = NOW() WHERE id = $1 +` + +func (q *Queries) TouchServiceDomainChecked(ctx context.Context, id uuid.UUID) error { + _, err := q.db.ExecContext(ctx, touchServiceDomainChecked, id) + return err +} diff --git a/app/backend/internal/database/sqlcdb/metrics.sql.go b/app/backend/internal/database/sqlcdb/metrics.sql.go new file mode 100644 index 0000000..63d69c9 --- /dev/null +++ b/app/backend/internal/database/sqlcdb/metrics.sql.go @@ -0,0 +1,623 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.31.1 +// source: metrics.sql + +package sqlcdb + +import ( + "context" + "database/sql" + "time" +) + +const aggregateNodeMetrics = `-- name: AggregateNodeMetrics :many +SELECT date_bin($1::interval, timestamp, '2000-01-01'::timestamptz) AS bucket, + AVG(cpu_usage)::float8 AS avg_cpu, + AVG(memory_usage_percent)::float8 AS avg_memory, + AVG(storage_usage_percent)::float8 AS avg_storage +FROM node_metrics +WHERE node_id = $2 AND timestamp BETWEEN $3 AND $4 +GROUP BY bucket +ORDER BY bucket ASC +` + +type AggregateNodeMetricsParams struct { + Column1 int64 `json:"column_1"` + NodeID string `json:"node_id"` + Timestamp time.Time `json:"timestamp"` + Timestamp_2 time.Time `json:"timestamp_2"` +} + +type AggregateNodeMetricsRow struct { + Bucket interface{} `json:"bucket"` + AvgCpu float64 `json:"avg_cpu"` + AvgMemory float64 `json:"avg_memory"` + AvgStorage float64 `json:"avg_storage"` +} + +// Portable bucket bucketing (no TimescaleDB): floor epoch seconds into +// bucket-sized buckets, then back to timestamptz. +func (q *Queries) AggregateNodeMetrics(ctx context.Context, arg AggregateNodeMetricsParams) ([]AggregateNodeMetricsRow, error) { + rows, err := q.db.QueryContext(ctx, aggregateNodeMetrics, + arg.Column1, + arg.NodeID, + arg.Timestamp, + arg.Timestamp_2, + ) + if err != nil { + return nil, err + } + defer rows.Close() + items := []AggregateNodeMetricsRow{} + for rows.Next() { + var i AggregateNodeMetricsRow + if err := rows.Scan( + &i.Bucket, + &i.AvgCpu, + &i.AvgMemory, + &i.AvgStorage, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const listInstanceMetrics = `-- name: ListInstanceMetrics :many +SELECT instance_id, node_id, status, cpu, memory, + network_bytes_in, network_bytes_out, network_packets_in, network_packets_out, + network_connections_in, network_connections_out, network_errors_in, network_errors_out, + start_time, last_seen, health_status, health_last_check, health_check_count, health_failure_count +FROM instance_metrics +WHERE service_id = $1 AND timestamp = $2 +` + +type ListInstanceMetricsParams struct { + ServiceID string `json:"service_id"` + Timestamp time.Time `json:"timestamp"` +} + +type ListInstanceMetricsRow struct { + InstanceID string `json:"instance_id"` + NodeID sql.NullString `json:"node_id"` + Status sql.NullString `json:"status"` + Cpu sql.NullString `json:"cpu"` + Memory sql.NullInt64 `json:"memory"` + NetworkBytesIn sql.NullInt64 `json:"network_bytes_in"` + NetworkBytesOut sql.NullInt64 `json:"network_bytes_out"` + NetworkPacketsIn sql.NullInt64 `json:"network_packets_in"` + NetworkPacketsOut sql.NullInt64 `json:"network_packets_out"` + NetworkConnectionsIn sql.NullInt32 `json:"network_connections_in"` + NetworkConnectionsOut sql.NullInt32 `json:"network_connections_out"` + NetworkErrorsIn sql.NullInt64 `json:"network_errors_in"` + NetworkErrorsOut sql.NullInt64 `json:"network_errors_out"` + StartTime sql.NullTime `json:"start_time"` + LastSeen sql.NullTime `json:"last_seen"` + HealthStatus sql.NullString `json:"health_status"` + HealthLastCheck sql.NullTime `json:"health_last_check"` + HealthCheckCount sql.NullInt32 `json:"health_check_count"` + HealthFailureCount sql.NullInt32 `json:"health_failure_count"` +} + +func (q *Queries) ListInstanceMetrics(ctx context.Context, arg ListInstanceMetricsParams) ([]ListInstanceMetricsRow, error) { + rows, err := q.db.QueryContext(ctx, listInstanceMetrics, arg.ServiceID, arg.Timestamp) + if err != nil { + return nil, err + } + defer rows.Close() + items := []ListInstanceMetricsRow{} + for rows.Next() { + var i ListInstanceMetricsRow + if err := rows.Scan( + &i.InstanceID, + &i.NodeID, + &i.Status, + &i.Cpu, + &i.Memory, + &i.NetworkBytesIn, + &i.NetworkBytesOut, + &i.NetworkPacketsIn, + &i.NetworkPacketsOut, + &i.NetworkConnectionsIn, + &i.NetworkConnectionsOut, + &i.NetworkErrorsIn, + &i.NetworkErrorsOut, + &i.StartTime, + &i.LastSeen, + &i.HealthStatus, + &i.HealthLastCheck, + &i.HealthCheckCount, + &i.HealthFailureCount, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const listNodeMetrics = `-- name: ListNodeMetrics :many +SELECT node_id, timestamp, cpu_usage, cpu_cores, load_avg_1, load_avg_5, load_avg_15, + memory_total, memory_used, memory_available, memory_usage_percent, + storage_total, storage_used, storage_available, storage_usage_percent, + network_bytes_in, network_bytes_out, network_packets_in, network_packets_out, + network_connections_in, network_connections_out, network_errors_in, network_errors_out, + coalesce(extract(epoch from uptime), 0)::bigint AS uptime_seconds, + processes, os, kernel, architecture +FROM node_metrics +WHERE node_id = $1 AND timestamp BETWEEN $2 AND $3 +ORDER BY timestamp ASC +` + +type ListNodeMetricsParams struct { + NodeID string `json:"node_id"` + Timestamp time.Time `json:"timestamp"` + Timestamp_2 time.Time `json:"timestamp_2"` +} + +type ListNodeMetricsRow struct { + NodeID string `json:"node_id"` + Timestamp time.Time `json:"timestamp"` + CpuUsage sql.NullString `json:"cpu_usage"` + CpuCores sql.NullString `json:"cpu_cores"` + LoadAvg1 sql.NullString `json:"load_avg_1"` + LoadAvg5 sql.NullString `json:"load_avg_5"` + LoadAvg15 sql.NullString `json:"load_avg_15"` + MemoryTotal sql.NullInt64 `json:"memory_total"` + MemoryUsed sql.NullInt64 `json:"memory_used"` + MemoryAvailable sql.NullInt64 `json:"memory_available"` + MemoryUsagePercent sql.NullString `json:"memory_usage_percent"` + StorageTotal sql.NullInt64 `json:"storage_total"` + StorageUsed sql.NullInt64 `json:"storage_used"` + StorageAvailable sql.NullInt64 `json:"storage_available"` + StorageUsagePercent sql.NullString `json:"storage_usage_percent"` + NetworkBytesIn sql.NullInt64 `json:"network_bytes_in"` + NetworkBytesOut sql.NullInt64 `json:"network_bytes_out"` + NetworkPacketsIn sql.NullInt64 `json:"network_packets_in"` + NetworkPacketsOut sql.NullInt64 `json:"network_packets_out"` + NetworkConnectionsIn sql.NullInt32 `json:"network_connections_in"` + NetworkConnectionsOut sql.NullInt32 `json:"network_connections_out"` + NetworkErrorsIn sql.NullInt64 `json:"network_errors_in"` + NetworkErrorsOut sql.NullInt64 `json:"network_errors_out"` + UptimeSeconds int64 `json:"uptime_seconds"` + Processes sql.NullInt32 `json:"processes"` + Os sql.NullString `json:"os"` + Kernel sql.NullString `json:"kernel"` + Architecture sql.NullString `json:"architecture"` +} + +func (q *Queries) ListNodeMetrics(ctx context.Context, arg ListNodeMetricsParams) ([]ListNodeMetricsRow, error) { + rows, err := q.db.QueryContext(ctx, listNodeMetrics, arg.NodeID, arg.Timestamp, arg.Timestamp_2) + if err != nil { + return nil, err + } + defer rows.Close() + items := []ListNodeMetricsRow{} + for rows.Next() { + var i ListNodeMetricsRow + if err := rows.Scan( + &i.NodeID, + &i.Timestamp, + &i.CpuUsage, + &i.CpuCores, + &i.LoadAvg1, + &i.LoadAvg5, + &i.LoadAvg15, + &i.MemoryTotal, + &i.MemoryUsed, + &i.MemoryAvailable, + &i.MemoryUsagePercent, + &i.StorageTotal, + &i.StorageUsed, + &i.StorageAvailable, + &i.StorageUsagePercent, + &i.NetworkBytesIn, + &i.NetworkBytesOut, + &i.NetworkPacketsIn, + &i.NetworkPacketsOut, + &i.NetworkConnectionsIn, + &i.NetworkConnectionsOut, + &i.NetworkErrorsIn, + &i.NetworkErrorsOut, + &i.UptimeSeconds, + &i.Processes, + &i.Os, + &i.Kernel, + &i.Architecture, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const listServiceMetrics = `-- name: ListServiceMetrics :many +SELECT service_id, service_name, project_id, timestamp, + requests_total, requests_success, requests_errors, requests_avg_latency, + requests_p95_latency, requests_p99_latency, requests_throughput, + errors_total, errors_rate, performance_response_time, performance_throughput, + performance_concurrency, performance_saturation, performance_utilization, + resource_cpu_usage, resource_memory_usage, resource_storage_usage, + resource_network_usage, resource_score +FROM service_metrics +WHERE service_id = $1 AND timestamp BETWEEN $2 AND $3 +ORDER BY timestamp ASC +` + +type ListServiceMetricsParams struct { + ServiceID string `json:"service_id"` + Timestamp time.Time `json:"timestamp"` + Timestamp_2 time.Time `json:"timestamp_2"` +} + +type ListServiceMetricsRow struct { + ServiceID string `json:"service_id"` + ServiceName string `json:"service_name"` + ProjectID string `json:"project_id"` + Timestamp time.Time `json:"timestamp"` + RequestsTotal sql.NullInt64 `json:"requests_total"` + RequestsSuccess sql.NullInt64 `json:"requests_success"` + RequestsErrors sql.NullInt64 `json:"requests_errors"` + RequestsAvgLatency sql.NullString `json:"requests_avg_latency"` + RequestsP95Latency sql.NullString `json:"requests_p95_latency"` + RequestsP99Latency sql.NullString `json:"requests_p99_latency"` + RequestsThroughput sql.NullString `json:"requests_throughput"` + ErrorsTotal sql.NullInt64 `json:"errors_total"` + ErrorsRate sql.NullString `json:"errors_rate"` + PerformanceResponseTime sql.NullString `json:"performance_response_time"` + PerformanceThroughput sql.NullString `json:"performance_throughput"` + PerformanceConcurrency sql.NullInt64 `json:"performance_concurrency"` + PerformanceSaturation sql.NullString `json:"performance_saturation"` + PerformanceUtilization sql.NullString `json:"performance_utilization"` + ResourceCpuUsage sql.NullString `json:"resource_cpu_usage"` + ResourceMemoryUsage sql.NullInt64 `json:"resource_memory_usage"` + ResourceStorageUsage sql.NullInt64 `json:"resource_storage_usage"` + ResourceNetworkUsage sql.NullInt64 `json:"resource_network_usage"` + ResourceScore sql.NullString `json:"resource_score"` +} + +func (q *Queries) ListServiceMetrics(ctx context.Context, arg ListServiceMetricsParams) ([]ListServiceMetricsRow, error) { + rows, err := q.db.QueryContext(ctx, listServiceMetrics, arg.ServiceID, arg.Timestamp, arg.Timestamp_2) + if err != nil { + return nil, err + } + defer rows.Close() + items := []ListServiceMetricsRow{} + for rows.Next() { + var i ListServiceMetricsRow + if err := rows.Scan( + &i.ServiceID, + &i.ServiceName, + &i.ProjectID, + &i.Timestamp, + &i.RequestsTotal, + &i.RequestsSuccess, + &i.RequestsErrors, + &i.RequestsAvgLatency, + &i.RequestsP95Latency, + &i.RequestsP99Latency, + &i.RequestsThroughput, + &i.ErrorsTotal, + &i.ErrorsRate, + &i.PerformanceResponseTime, + &i.PerformanceThroughput, + &i.PerformanceConcurrency, + &i.PerformanceSaturation, + &i.PerformanceUtilization, + &i.ResourceCpuUsage, + &i.ResourceMemoryUsage, + &i.ResourceStorageUsage, + &i.ResourceNetworkUsage, + &i.ResourceScore, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const upsertInstanceMetrics = `-- name: UpsertInstanceMetrics :exec +INSERT INTO instance_metrics ( + service_id, timestamp, instance_id, node_id, status, cpu, memory, + network_bytes_in, network_bytes_out, network_packets_in, network_packets_out, + network_connections_in, network_connections_out, network_errors_in, network_errors_out, + start_time, last_seen, health_status, health_last_check, health_check_count, health_failure_count +) VALUES ( + $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, + $16, $17, $18, $19, $20, $21 +) +ON CONFLICT (service_id, timestamp, instance_id) DO UPDATE SET + node_id = EXCLUDED.node_id, + status = EXCLUDED.status, + cpu = EXCLUDED.cpu, + memory = EXCLUDED.memory, + network_bytes_in = EXCLUDED.network_bytes_in, + network_bytes_out = EXCLUDED.network_bytes_out, + network_packets_in = EXCLUDED.network_packets_in, + network_packets_out = EXCLUDED.network_packets_out, + network_connections_in = EXCLUDED.network_connections_in, + network_connections_out = EXCLUDED.network_connections_out, + network_errors_in = EXCLUDED.network_errors_in, + network_errors_out = EXCLUDED.network_errors_out, + start_time = EXCLUDED.start_time, + last_seen = EXCLUDED.last_seen, + health_status = EXCLUDED.health_status, + health_last_check = EXCLUDED.health_last_check, + health_check_count = EXCLUDED.health_check_count, + health_failure_count = EXCLUDED.health_failure_count +` + +type UpsertInstanceMetricsParams struct { + ServiceID string `json:"service_id"` + Timestamp time.Time `json:"timestamp"` + InstanceID string `json:"instance_id"` + NodeID sql.NullString `json:"node_id"` + Status sql.NullString `json:"status"` + Cpu sql.NullString `json:"cpu"` + Memory sql.NullInt64 `json:"memory"` + NetworkBytesIn sql.NullInt64 `json:"network_bytes_in"` + NetworkBytesOut sql.NullInt64 `json:"network_bytes_out"` + NetworkPacketsIn sql.NullInt64 `json:"network_packets_in"` + NetworkPacketsOut sql.NullInt64 `json:"network_packets_out"` + NetworkConnectionsIn sql.NullInt32 `json:"network_connections_in"` + NetworkConnectionsOut sql.NullInt32 `json:"network_connections_out"` + NetworkErrorsIn sql.NullInt64 `json:"network_errors_in"` + NetworkErrorsOut sql.NullInt64 `json:"network_errors_out"` + StartTime sql.NullTime `json:"start_time"` + LastSeen sql.NullTime `json:"last_seen"` + HealthStatus sql.NullString `json:"health_status"` + HealthLastCheck sql.NullTime `json:"health_last_check"` + HealthCheckCount sql.NullInt32 `json:"health_check_count"` + HealthFailureCount sql.NullInt32 `json:"health_failure_count"` +} + +func (q *Queries) UpsertInstanceMetrics(ctx context.Context, arg UpsertInstanceMetricsParams) error { + _, err := q.db.ExecContext(ctx, upsertInstanceMetrics, + arg.ServiceID, + arg.Timestamp, + arg.InstanceID, + arg.NodeID, + arg.Status, + arg.Cpu, + arg.Memory, + arg.NetworkBytesIn, + arg.NetworkBytesOut, + arg.NetworkPacketsIn, + arg.NetworkPacketsOut, + arg.NetworkConnectionsIn, + arg.NetworkConnectionsOut, + arg.NetworkErrorsIn, + arg.NetworkErrorsOut, + arg.StartTime, + arg.LastSeen, + arg.HealthStatus, + arg.HealthLastCheck, + arg.HealthCheckCount, + arg.HealthFailureCount, + ) + return err +} + +const upsertNodeMetrics = `-- name: UpsertNodeMetrics :exec +INSERT INTO node_metrics ( + node_id, timestamp, cpu_usage, cpu_cores, load_avg_1, load_avg_5, load_avg_15, + memory_total, memory_used, memory_available, memory_usage_percent, + storage_total, storage_used, storage_available, storage_usage_percent, + network_bytes_in, network_bytes_out, network_packets_in, network_packets_out, + network_connections_in, network_connections_out, network_errors_in, network_errors_out, + uptime, processes, os, kernel, architecture +) VALUES ( + $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, + $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26, $27, $28 +) +ON CONFLICT (node_id, timestamp) DO UPDATE SET + cpu_usage = EXCLUDED.cpu_usage, + cpu_cores = EXCLUDED.cpu_cores, + load_avg_1 = EXCLUDED.load_avg_1, + load_avg_5 = EXCLUDED.load_avg_5, + load_avg_15 = EXCLUDED.load_avg_15, + memory_total = EXCLUDED.memory_total, + memory_used = EXCLUDED.memory_used, + memory_available = EXCLUDED.memory_available, + memory_usage_percent = EXCLUDED.memory_usage_percent, + storage_total = EXCLUDED.storage_total, + storage_used = EXCLUDED.storage_used, + storage_available = EXCLUDED.storage_available, + storage_usage_percent = EXCLUDED.storage_usage_percent, + network_bytes_in = EXCLUDED.network_bytes_in, + network_bytes_out = EXCLUDED.network_bytes_out, + network_packets_in = EXCLUDED.network_packets_in, + network_packets_out = EXCLUDED.network_packets_out, + network_connections_in = EXCLUDED.network_connections_in, + network_connections_out = EXCLUDED.network_connections_out, + network_errors_in = EXCLUDED.network_errors_in, + network_errors_out = EXCLUDED.network_errors_out, + uptime = EXCLUDED.uptime, + processes = EXCLUDED.processes, + os = EXCLUDED.os, + kernel = EXCLUDED.kernel, + architecture = EXCLUDED.architecture +` + +type UpsertNodeMetricsParams struct { + NodeID string `json:"node_id"` + Timestamp time.Time `json:"timestamp"` + CpuUsage sql.NullString `json:"cpu_usage"` + CpuCores sql.NullString `json:"cpu_cores"` + LoadAvg1 sql.NullString `json:"load_avg_1"` + LoadAvg5 sql.NullString `json:"load_avg_5"` + LoadAvg15 sql.NullString `json:"load_avg_15"` + MemoryTotal sql.NullInt64 `json:"memory_total"` + MemoryUsed sql.NullInt64 `json:"memory_used"` + MemoryAvailable sql.NullInt64 `json:"memory_available"` + MemoryUsagePercent sql.NullString `json:"memory_usage_percent"` + StorageTotal sql.NullInt64 `json:"storage_total"` + StorageUsed sql.NullInt64 `json:"storage_used"` + StorageAvailable sql.NullInt64 `json:"storage_available"` + StorageUsagePercent sql.NullString `json:"storage_usage_percent"` + NetworkBytesIn sql.NullInt64 `json:"network_bytes_in"` + NetworkBytesOut sql.NullInt64 `json:"network_bytes_out"` + NetworkPacketsIn sql.NullInt64 `json:"network_packets_in"` + NetworkPacketsOut sql.NullInt64 `json:"network_packets_out"` + NetworkConnectionsIn sql.NullInt32 `json:"network_connections_in"` + NetworkConnectionsOut sql.NullInt32 `json:"network_connections_out"` + NetworkErrorsIn sql.NullInt64 `json:"network_errors_in"` + NetworkErrorsOut sql.NullInt64 `json:"network_errors_out"` + Uptime sql.NullInt64 `json:"uptime"` + Processes sql.NullInt32 `json:"processes"` + Os sql.NullString `json:"os"` + Kernel sql.NullString `json:"kernel"` + Architecture sql.NullString `json:"architecture"` +} + +func (q *Queries) UpsertNodeMetrics(ctx context.Context, arg UpsertNodeMetricsParams) error { + _, err := q.db.ExecContext(ctx, upsertNodeMetrics, + arg.NodeID, + arg.Timestamp, + arg.CpuUsage, + arg.CpuCores, + arg.LoadAvg1, + arg.LoadAvg5, + arg.LoadAvg15, + arg.MemoryTotal, + arg.MemoryUsed, + arg.MemoryAvailable, + arg.MemoryUsagePercent, + arg.StorageTotal, + arg.StorageUsed, + arg.StorageAvailable, + arg.StorageUsagePercent, + arg.NetworkBytesIn, + arg.NetworkBytesOut, + arg.NetworkPacketsIn, + arg.NetworkPacketsOut, + arg.NetworkConnectionsIn, + arg.NetworkConnectionsOut, + arg.NetworkErrorsIn, + arg.NetworkErrorsOut, + arg.Uptime, + arg.Processes, + arg.Os, + arg.Kernel, + arg.Architecture, + ) + return err +} + +const upsertServiceMetrics = `-- name: UpsertServiceMetrics :exec +INSERT INTO service_metrics ( + service_id, service_name, project_id, timestamp, + requests_total, requests_success, requests_errors, requests_avg_latency, + requests_p95_latency, requests_p99_latency, requests_throughput, + errors_total, errors_rate, performance_response_time, performance_throughput, + performance_concurrency, performance_saturation, performance_utilization, + resource_cpu_usage, resource_memory_usage, resource_storage_usage, + resource_network_usage, resource_score +) VALUES ( + $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, + $16, $17, $18, $19, $20, $21, $22, $23 +) +ON CONFLICT (service_id, timestamp) DO UPDATE SET + requests_total = EXCLUDED.requests_total, + requests_success = EXCLUDED.requests_success, + requests_errors = EXCLUDED.requests_errors, + requests_avg_latency = EXCLUDED.requests_avg_latency, + requests_p95_latency = EXCLUDED.requests_p95_latency, + requests_p99_latency = EXCLUDED.requests_p99_latency, + requests_throughput = EXCLUDED.requests_throughput, + errors_total = EXCLUDED.errors_total, + errors_rate = EXCLUDED.errors_rate, + performance_response_time = EXCLUDED.performance_response_time, + performance_throughput = EXCLUDED.performance_throughput, + performance_concurrency = EXCLUDED.performance_concurrency, + performance_saturation = EXCLUDED.performance_saturation, + performance_utilization = EXCLUDED.performance_utilization, + resource_cpu_usage = EXCLUDED.resource_cpu_usage, + resource_memory_usage = EXCLUDED.resource_memory_usage, + resource_storage_usage = EXCLUDED.resource_storage_usage, + resource_network_usage = EXCLUDED.resource_network_usage, + resource_score = EXCLUDED.resource_score +` + +type UpsertServiceMetricsParams struct { + ServiceID string `json:"service_id"` + ServiceName string `json:"service_name"` + ProjectID string `json:"project_id"` + Timestamp time.Time `json:"timestamp"` + RequestsTotal sql.NullInt64 `json:"requests_total"` + RequestsSuccess sql.NullInt64 `json:"requests_success"` + RequestsErrors sql.NullInt64 `json:"requests_errors"` + RequestsAvgLatency sql.NullString `json:"requests_avg_latency"` + RequestsP95Latency sql.NullString `json:"requests_p95_latency"` + RequestsP99Latency sql.NullString `json:"requests_p99_latency"` + RequestsThroughput sql.NullString `json:"requests_throughput"` + ErrorsTotal sql.NullInt64 `json:"errors_total"` + ErrorsRate sql.NullString `json:"errors_rate"` + PerformanceResponseTime sql.NullString `json:"performance_response_time"` + PerformanceThroughput sql.NullString `json:"performance_throughput"` + PerformanceConcurrency sql.NullInt64 `json:"performance_concurrency"` + PerformanceSaturation sql.NullString `json:"performance_saturation"` + PerformanceUtilization sql.NullString `json:"performance_utilization"` + ResourceCpuUsage sql.NullString `json:"resource_cpu_usage"` + ResourceMemoryUsage sql.NullInt64 `json:"resource_memory_usage"` + ResourceStorageUsage sql.NullInt64 `json:"resource_storage_usage"` + ResourceNetworkUsage sql.NullInt64 `json:"resource_network_usage"` + ResourceScore sql.NullString `json:"resource_score"` +} + +func (q *Queries) UpsertServiceMetrics(ctx context.Context, arg UpsertServiceMetricsParams) error { + _, err := q.db.ExecContext(ctx, upsertServiceMetrics, + arg.ServiceID, + arg.ServiceName, + arg.ProjectID, + arg.Timestamp, + arg.RequestsTotal, + arg.RequestsSuccess, + arg.RequestsErrors, + arg.RequestsAvgLatency, + arg.RequestsP95Latency, + arg.RequestsP99Latency, + arg.RequestsThroughput, + arg.ErrorsTotal, + arg.ErrorsRate, + arg.PerformanceResponseTime, + arg.PerformanceThroughput, + arg.PerformanceConcurrency, + arg.PerformanceSaturation, + arg.PerformanceUtilization, + arg.ResourceCpuUsage, + arg.ResourceMemoryUsage, + arg.ResourceStorageUsage, + arg.ResourceNetworkUsage, + arg.ResourceScore, + ) + return err +} diff --git a/app/backend/internal/database/sqlcdb/models.go b/app/backend/internal/database/sqlcdb/models.go index c91e277..0a170c1 100644 --- a/app/backend/internal/database/sqlcdb/models.go +++ b/app/backend/internal/database/sqlcdb/models.go @@ -161,6 +161,31 @@ type EnvironmentVariable struct { UpdatedAt sql.NullTime `json:"updated_at"` } +type InstanceMetric struct { + ServiceID string `json:"service_id"` + Timestamp time.Time `json:"timestamp"` + InstanceID string `json:"instance_id"` + NodeID sql.NullString `json:"node_id"` + Status sql.NullString `json:"status"` + Cpu sql.NullString `json:"cpu"` + Memory sql.NullInt64 `json:"memory"` + NetworkBytesIn sql.NullInt64 `json:"network_bytes_in"` + NetworkBytesOut sql.NullInt64 `json:"network_bytes_out"` + NetworkPacketsIn sql.NullInt64 `json:"network_packets_in"` + NetworkPacketsOut sql.NullInt64 `json:"network_packets_out"` + NetworkConnectionsIn sql.NullInt32 `json:"network_connections_in"` + NetworkConnectionsOut sql.NullInt32 `json:"network_connections_out"` + NetworkErrorsIn sql.NullInt64 `json:"network_errors_in"` + NetworkErrorsOut sql.NullInt64 `json:"network_errors_out"` + StartTime sql.NullTime `json:"start_time"` + LastSeen sql.NullTime `json:"last_seen"` + HealthStatus sql.NullString `json:"health_status"` + HealthLastCheck sql.NullTime `json:"health_last_check"` + HealthCheckCount sql.NullInt32 `json:"health_check_count"` + HealthFailureCount sql.NullInt32 `json:"health_failure_count"` + CreatedAt sql.NullTime `json:"created_at"` +} + type NodeAgent struct { ID string `json:"id"` Name string `json:"name"` @@ -181,6 +206,38 @@ type NodeAgent struct { DefaultDomain string `json:"default_domain"` } +type NodeMetric struct { + NodeID string `json:"node_id"` + Timestamp time.Time `json:"timestamp"` + CpuUsage sql.NullString `json:"cpu_usage"` + CpuCores sql.NullString `json:"cpu_cores"` + LoadAvg1 sql.NullString `json:"load_avg_1"` + LoadAvg5 sql.NullString `json:"load_avg_5"` + LoadAvg15 sql.NullString `json:"load_avg_15"` + MemoryTotal sql.NullInt64 `json:"memory_total"` + MemoryUsed sql.NullInt64 `json:"memory_used"` + MemoryAvailable sql.NullInt64 `json:"memory_available"` + MemoryUsagePercent sql.NullString `json:"memory_usage_percent"` + StorageTotal sql.NullInt64 `json:"storage_total"` + StorageUsed sql.NullInt64 `json:"storage_used"` + StorageAvailable sql.NullInt64 `json:"storage_available"` + StorageUsagePercent sql.NullString `json:"storage_usage_percent"` + NetworkBytesIn sql.NullInt64 `json:"network_bytes_in"` + NetworkBytesOut sql.NullInt64 `json:"network_bytes_out"` + NetworkPacketsIn sql.NullInt64 `json:"network_packets_in"` + NetworkPacketsOut sql.NullInt64 `json:"network_packets_out"` + NetworkConnectionsIn sql.NullInt32 `json:"network_connections_in"` + NetworkConnectionsOut sql.NullInt32 `json:"network_connections_out"` + NetworkErrorsIn sql.NullInt64 `json:"network_errors_in"` + NetworkErrorsOut sql.NullInt64 `json:"network_errors_out"` + Uptime sql.NullInt64 `json:"uptime"` + Processes sql.NullInt32 `json:"processes"` + Os sql.NullString `json:"os"` + Kernel sql.NullString `json:"kernel"` + Architecture sql.NullString `json:"architecture"` + CreatedAt sql.NullTime `json:"created_at"` +} + type Notification struct { ID uuid.UUID `json:"id"` UserID uuid.UUID `json:"user_id"` @@ -305,6 +362,33 @@ type ServiceDomain struct { CreatedAt time.Time `json:"created_at"` } +type ServiceMetric struct { + ServiceID string `json:"service_id"` + ServiceName string `json:"service_name"` + ProjectID string `json:"project_id"` + Timestamp time.Time `json:"timestamp"` + RequestsTotal sql.NullInt64 `json:"requests_total"` + RequestsSuccess sql.NullInt64 `json:"requests_success"` + RequestsErrors sql.NullInt64 `json:"requests_errors"` + RequestsAvgLatency sql.NullString `json:"requests_avg_latency"` + RequestsP95Latency sql.NullString `json:"requests_p95_latency"` + RequestsP99Latency sql.NullString `json:"requests_p99_latency"` + RequestsThroughput sql.NullString `json:"requests_throughput"` + ErrorsTotal sql.NullInt64 `json:"errors_total"` + ErrorsRate sql.NullString `json:"errors_rate"` + PerformanceResponseTime sql.NullString `json:"performance_response_time"` + PerformanceThroughput sql.NullString `json:"performance_throughput"` + PerformanceConcurrency sql.NullInt64 `json:"performance_concurrency"` + PerformanceSaturation sql.NullString `json:"performance_saturation"` + PerformanceUtilization sql.NullString `json:"performance_utilization"` + ResourceCpuUsage sql.NullString `json:"resource_cpu_usage"` + ResourceMemoryUsage sql.NullInt64 `json:"resource_memory_usage"` + ResourceStorageUsage sql.NullInt64 `json:"resource_storage_usage"` + ResourceNetworkUsage sql.NullInt64 `json:"resource_network_usage"` + ResourceScore sql.NullString `json:"resource_score"` + CreatedAt sql.NullTime `json:"created_at"` +} + type ServiceTemplate struct { ID string `json:"id"` Name string `json:"name"` diff --git a/app/backend/internal/database/sqlcdb/querier.go b/app/backend/internal/database/sqlcdb/querier.go index 8fafab2..0700f84 100644 --- a/app/backend/internal/database/sqlcdb/querier.go +++ b/app/backend/internal/database/sqlcdb/querier.go @@ -13,12 +13,17 @@ import ( ) type Querier interface { + // Portable bucket bucketing (no TimescaleDB): floor epoch seconds into + // bucket-sized buckets, then back to timestamptz. + AggregateNodeMetrics(ctx context.Context, arg AggregateNodeMetricsParams) ([]AggregateNodeMetricsRow, error) + ClearServiceDomainDefault(ctx context.Context, serviceID uuid.UUID) error ClearServiceNodePins(ctx context.Context, nodeID sql.NullString) (int64, error) CompleteCommand(ctx context.Context, arg CompleteCommandParams) (AgentCommand, error) CountDatabaseServicesByUserAndName(ctx context.Context, arg CountDatabaseServicesByUserAndNameParams) (int64, error) CountDatabasesUsingBackupTarget(ctx context.Context, targetID sql.NullString) (int64, error) CountEnvironmentServices(ctx context.Context, environmentID uuid.UUID) (int64, error) CountProjectsByUser(ctx context.Context, arg CountProjectsByUserParams) (int64, error) + CountServiceDomains(ctx context.Context, serviceID uuid.UUID) (int64, error) CountServicesByProjectAndName(ctx context.Context, arg CountServicesByProjectAndNameParams) (int64, error) CountUnreadNotificationsByUser(ctx context.Context, userID uuid.UUID) (int64, error) CreateAgent(ctx context.Context, arg CreateAgentParams) (NodeAgent, error) @@ -33,6 +38,7 @@ type Querier interface { CreateNotification(ctx context.Context, arg CreateNotificationParams) error CreateOutboundWebhook(ctx context.Context, arg CreateOutboundWebhookParams) error CreateProject(ctx context.Context, arg CreateProjectParams) (Project, error) + CreateServiceDomain(ctx context.Context, arg CreateServiceDomainParams) (CreateServiceDomainRow, error) CreateServiceFromTemplate(ctx context.Context, arg CreateServiceFromTemplateParams) error CreateUserInvite(ctx context.Context, arg CreateUserInviteParams) (UserInvite, error) CreateUserTemplate(ctx context.Context, arg CreateUserTemplateParams) error @@ -49,6 +55,7 @@ type Querier interface { DeleteProjectByID(ctx context.Context, projectID uuid.UUID) (int64, error) DeleteProjectEnvironment(ctx context.Context, arg DeleteProjectEnvironmentParams) error DeleteServiceContainersOnAgent(ctx context.Context, arg DeleteServiceContainersOnAgentParams) error + DeleteServiceDomain(ctx context.Context, arg DeleteServiceDomainParams) (int64, error) DeleteUserInvite(ctx context.Context, id uuid.UUID) error DeleteUserTemplate(ctx context.Context, arg DeleteUserTemplateParams) (int64, error) GetActiveAgentAuthTokenByHash(ctx context.Context, tokenHash string) (AgentAuthToken, error) @@ -72,6 +79,10 @@ type Querier interface { GetProjectOwnerByID(ctx context.Context, projectID uuid.UUID) (uuid.UUID, error) GetProjectOwnerID(ctx context.Context, id uuid.UUID) (uuid.UUID, error) GetProjectRoleForUser(ctx context.Context, arg GetProjectRoleForUserParams) (string, error) + GetServiceAccess(ctx context.Context, id uuid.UUID) (GetServiceAccessRow, error) + GetServiceDomainDefault(ctx context.Context, serviceID uuid.UUID) (string, error) + GetServiceDomainFirst(ctx context.Context, serviceID uuid.UUID) (string, error) + GetServicePlacementBrief(ctx context.Context, id uuid.UUID) (GetServicePlacementBriefRow, error) GetServiceTemplateByID(ctx context.Context, id string) (ServiceTemplate, error) GetUserInviteByTokenHash(ctx context.Context, tokenHash string) (UserInvite, error) GetUserTokenByHash(ctx context.Context, tokenHash string) (UserToken, error) @@ -93,6 +104,8 @@ type Querier interface { ListDatabaseServicesByUser(ctx context.Context, userID string) ([]DatabaseService, error) ListDueDatabaseBackups(ctx context.Context) ([]ListDueDatabaseBackupsRow, error) ListEnabledOutboundWebhooks(ctx context.Context) ([]OutboundWebhook, error) + ListInstanceMetrics(ctx context.Context, arg ListInstanceMetricsParams) ([]ListInstanceMetricsRow, error) + ListNodeMetrics(ctx context.Context, arg ListNodeMetricsParams) ([]ListNodeMetricsRow, error) ListNotificationChannelsByUser(ctx context.Context, userID uuid.UUID) ([]NotificationChannel, error) ListNotificationsByUser(ctx context.Context, arg ListNotificationsByUserParams) ([]Notification, error) ListOutboundWebhooksByUser(ctx context.Context, userID uuid.UUID) ([]OutboundWebhook, error) @@ -112,6 +125,8 @@ type Querier interface { // Distinct agents holding inventory rows for a service. ListServiceAgents(ctx context.Context, serviceID string) ([]string, error) ListServiceContainers(ctx context.Context, serviceID string) ([]ListServiceContainersRow, error) + ListServiceDomains(ctx context.Context, serviceID uuid.UUID) ([]ServiceDomain, error) + ListServiceMetrics(ctx context.Context, arg ListServiceMetricsParams) ([]ListServiceMetricsRow, error) ListServiceTemplatesByCategoryForUser(ctx context.Context, arg ListServiceTemplatesByCategoryForUserParams) ([]ServiceTemplate, error) ListServiceTemplatesForUser(ctx context.Context, ownerID uuid.NullUUID) ([]ServiceTemplate, error) ListUserInvites(ctx context.Context) ([]UserInvite, error) @@ -129,6 +144,7 @@ type Querier interface { RevokeAgentAuthToken(ctx context.Context, id uuid.UUID) (AgentAuthToken, error) RevokeUserToken(ctx context.Context, arg RevokeUserTokenParams) (int64, error) ScrubCommandPayload(ctx context.Context, arg ScrubCommandPayloadParams) error + ServiceWriteAccess(ctx context.Context, arg ServiceWriteAccessParams) (bool, error) SetAgentDefaultDomain(ctx context.Context, arg SetAgentDefaultDomainParams) error SetAgentSchedulable(ctx context.Context, arg SetAgentSchedulableParams) error SetAgentTags(ctx context.Context, arg SetAgentTagsParams) error @@ -142,7 +158,10 @@ type Querier interface { SetDatabaseServiceStatusByID(ctx context.Context, arg SetDatabaseServiceStatusByIDParams) error SetDatabaseServiceStatusByIDAndUser(ctx context.Context, arg SetDatabaseServiceStatusByIDAndUserParams) error SetProjectApproved(ctx context.Context, arg SetProjectApprovedParams) (int64, error) + SetServiceDomainDefault(ctx context.Context, arg SetServiceDomainDefaultParams) (int64, error) + SetServiceLegacyDomain(ctx context.Context, arg SetServiceLegacyDomainParams) error TouchAgentAuthToken(ctx context.Context, tokenHash string) error + TouchServiceDomainChecked(ctx context.Context, id uuid.UUID) error TouchUserToken(ctx context.Context, id uuid.UUID) error UpdateAgent(ctx context.Context, arg UpdateAgentParams) (NodeAgent, error) UpdateAgentHeartbeat(ctx context.Context, arg UpdateAgentHeartbeatParams) error @@ -164,7 +183,10 @@ type Querier interface { UpdateUserTemplate(ctx context.Context, arg UpdateUserTemplateParams) (int64, error) UpdateWebhookDeliveryResult(ctx context.Context, arg UpdateWebhookDeliveryResultParams) error UpsertEnvironmentVariable(ctx context.Context, arg UpsertEnvironmentVariableParams) error + UpsertInstanceMetrics(ctx context.Context, arg UpsertInstanceMetricsParams) error + UpsertNodeMetrics(ctx context.Context, arg UpsertNodeMetricsParams) error UpsertServiceContainer(ctx context.Context, arg UpsertServiceContainerParams) error + UpsertServiceMetrics(ctx context.Context, arg UpsertServiceMetricsParams) error UpsertServiceTemplate(ctx context.Context, arg UpsertServiceTemplateParams) error } diff --git a/app/backend/internal/metrics/storage.go b/app/backend/internal/metrics/storage.go index 7d376bd..68dc567 100644 --- a/app/backend/internal/metrics/storage.go +++ b/app/backend/internal/metrics/storage.go @@ -4,293 +4,286 @@ import ( "context" "database/sql" "fmt" + "strconv" "sync" "time" + "containr/internal/database/sqlcdb" + _ "github.com/lib/pq" ) // PostgreSQLMetricsStorage implements MetricsStorage using PostgreSQL +// through sqlc-generated queries (sqlc/queries/metrics.sql). +// +// Per-container node metrics are deliberately not persisted: the +// container_metrics table belongs to the container_instances model and +// has no place for the node/container samples collected here. Node and +// service/instance rows map cleanly, so those are stored. type PostgreSQLMetricsStorage struct { - db *sql.DB + q *sqlcdb.Queries } // NewPostgreSQLMetricsStorage creates a new PostgreSQL metrics storage func NewPostgreSQLMetricsStorage(db *sql.DB) *PostgreSQLMetricsStorage { - return &PostgreSQLMetricsStorage{db: db} + return &PostgreSQLMetricsStorage{q: sqlcdb.New(db)} } +func numStr(v float64) sql.NullString { + return sql.NullString{String: strconv.FormatFloat(v, 'f', -1, 64), Valid: true} +} + +func int64N(v int64) sql.NullInt64 { + return sql.NullInt64{Int64: v, Valid: true} +} + +func int32N(v int64) sql.NullInt32 { + return sql.NullInt32{Int32: int32(v), Valid: true} +} + +func strN(v string) sql.NullString { + return sql.NullString{String: v, Valid: v != ""} +} + +func timeN(t time.Time) sql.NullTime { + return sql.NullTime{Time: t, Valid: !t.IsZero()} +} + +func f64v(n sql.NullString) float64 { + f, _ := strconv.ParseFloat(n.String, 64) + return f +} + +func i64v(n sql.NullInt64) int64 { return n.Int64 } + +func i32v(n sql.NullInt32) int64 { return int64(n.Int32) } + // StoreNodeMetrics stores node metrics in the database func (s *PostgreSQLMetricsStorage) StoreNodeMetrics(ctx context.Context, metrics *NodeMetrics) error { - query := ` - INSERT INTO node_metrics ( - node_id, timestamp, cpu_usage, cpu_cores, load_avg_1, load_avg_5, load_avg_15, - memory_total, memory_used, memory_available, memory_usage_percent, - storage_total, storage_used, storage_available, storage_usage_percent, - network_bytes_in, network_bytes_out, network_packets_in, network_packets_out, - network_connections_in, network_connections_out, network_errors_in, network_errors_out, - uptime, processes, os, kernel, architecture - ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26, $27, $28) - ON CONFLICT (node_id, timestamp) DO UPDATE SET - cpu_usage = EXCLUDED.cpu_usage, - cpu_cores = EXCLUDED.cpu_cores, - load_avg_1 = EXCLUDED.load_avg_1, - load_avg_5 = EXCLUDED.load_avg_5, - load_avg_15 = EXCLUDED.load_avg_15, - memory_total = EXCLUDED.memory_total, - memory_used = EXCLUDED.memory_used, - memory_available = EXCLUDED.memory_available, - memory_usage_percent = EXCLUDED.memory_usage_percent, - storage_total = EXCLUDED.storage_total, - storage_used = EXCLUDED.storage_used, - storage_available = EXCLUDED.storage_available, - storage_usage_percent = EXCLUDED.storage_usage_percent, - network_bytes_in = EXCLUDED.network_bytes_in, - network_bytes_out = EXCLUDED.network_bytes_out, - network_packets_in = EXCLUDED.network_packets_in, - network_packets_out = EXCLUDED.network_packets_out, - network_connections_in = EXCLUDED.network_connections_in, - network_connections_out = EXCLUDED.network_connections_out, - network_errors_in = EXCLUDED.network_errors_in, - network_errors_out = EXCLUDED.network_errors_out, - uptime = EXCLUDED.uptime, - processes = EXCLUDED.processes, - os = EXCLUDED.os, - kernel = EXCLUDED.kernel, - architecture = EXCLUDED.architecture - ` - - _, err := s.db.ExecContext(ctx, query, - metrics.NodeID, metrics.Timestamp, metrics.CPU.UsagePercent, metrics.CPU.UsageCores, - metrics.CPU.LoadAverage1, metrics.CPU.LoadAverage5, metrics.CPU.LoadAverage15, - metrics.Memory.Total, metrics.Memory.Used, metrics.Memory.Available, metrics.Memory.UsagePercent, - metrics.Storage.Total, metrics.Storage.Used, metrics.Storage.Available, metrics.Storage.UsagePercent, - metrics.Network.BytesIn, metrics.Network.BytesOut, metrics.Network.PacketsIn, metrics.Network.PacketsOut, - metrics.Network.ConnectionsIn, metrics.Network.ConnectionsOut, metrics.Network.ErrorsIn, metrics.Network.ErrorsOut, - metrics.System.Uptime, metrics.System.Processes, metrics.System.OS, metrics.System.Kernel, metrics.System.Architecture, - ) - + err := s.q.UpsertNodeMetrics(ctx, sqlcdb.UpsertNodeMetricsParams{ + NodeID: metrics.NodeID, + Timestamp: metrics.Timestamp, + CpuUsage: numStr(metrics.CPU.UsagePercent), + CpuCores: numStr(metrics.CPU.UsageCores), + LoadAvg1: numStr(metrics.CPU.LoadAverage1), + LoadAvg5: numStr(metrics.CPU.LoadAverage5), + LoadAvg15: numStr(metrics.CPU.LoadAverage15), + MemoryTotal: int64N(metrics.Memory.Total), + MemoryUsed: int64N(metrics.Memory.Used), + MemoryAvailable: int64N(metrics.Memory.Available), + MemoryUsagePercent: numStr(metrics.Memory.UsagePercent), + StorageTotal: int64N(metrics.Storage.Total), + StorageUsed: int64N(metrics.Storage.Used), + StorageAvailable: int64N(metrics.Storage.Available), + StorageUsagePercent: numStr(metrics.Storage.UsagePercent), + NetworkBytesIn: int64N(metrics.Network.BytesIn), + NetworkBytesOut: int64N(metrics.Network.BytesOut), + NetworkPacketsIn: int64N(metrics.Network.PacketsIn), + NetworkPacketsOut: int64N(metrics.Network.PacketsOut), + NetworkConnectionsIn: int32N(metrics.Network.ConnectionsIn), + NetworkConnectionsOut: int32N(metrics.Network.ConnectionsOut), + NetworkErrorsIn: int64N(metrics.Network.ErrorsIn), + NetworkErrorsOut: int64N(metrics.Network.ErrorsOut), + // INTERVAL param — Postgres casts the bigint as seconds. + Uptime: int64N(int64(metrics.System.Uptime.Seconds())), + Processes: sql.NullInt32{Int32: int32(metrics.System.Processes), Valid: true}, + Os: strN(metrics.System.OS), + Kernel: strN(metrics.System.Kernel), + Architecture: strN(metrics.System.Architecture), + }) if err != nil { return fmt.Errorf("failed to store node metrics: %w", err) } - - // Store container metrics - for _, container := range metrics.Containers { - if err := s.storeContainerMetrics(ctx, metrics.NodeID, metrics.Timestamp, container); err != nil { - return fmt.Errorf("failed to store container metrics: %w", err) - } - } - return nil } // StoreServiceMetrics stores service metrics in the database func (s *PostgreSQLMetricsStorage) StoreServiceMetrics(ctx context.Context, metrics *ServiceMetrics) error { - query := ` - INSERT INTO service_metrics ( - service_id, service_name, project_id, timestamp, - requests_total, requests_success, requests_errors, requests_avg_latency, - requests_p95_latency, requests_p99_latency, requests_throughput, - errors_total, errors_rate, performance_response_time, performance_throughput, - performance_concurrency, performance_saturation, performance_utilization, - resource_cpu_usage, resource_memory_usage, resource_storage_usage, - resource_network_usage, resource_score - ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23) - ON CONFLICT (service_id, timestamp) DO UPDATE SET - requests_total = EXCLUDED.requests_total, - requests_success = EXCLUDED.requests_success, - requests_errors = EXCLUDED.requests_errors, - requests_avg_latency = EXCLUDED.requests_avg_latency, - requests_p95_latency = EXCLUDED.requests_p95_latency, - requests_p99_latency = EXCLUDED.requests_p99_latency, - requests_throughput = EXCLUDED.requests_throughput, - errors_total = EXCLUDED.errors_total, - errors_rate = EXCLUDED.errors_rate, - performance_response_time = EXCLUDED.performance_response_time, - performance_throughput = EXCLUDED.performance_throughput, - performance_concurrency = EXCLUDED.performance_concurrency, - performance_saturation = EXCLUDED.performance_saturation, - performance_utilization = EXCLUDED.performance_utilization, - resource_cpu_usage = EXCLUDED.resource_cpu_usage, - resource_memory_usage = EXCLUDED.resource_memory_usage, - resource_storage_usage = EXCLUDED.resource_storage_usage, - resource_network_usage = EXCLUDED.resource_network_usage, - resource_score = EXCLUDED.resource_score - ` - - _, err := s.db.ExecContext(ctx, query, - metrics.ServiceID, metrics.ServiceName, metrics.ProjectID, metrics.Timestamp, - metrics.Requests.Total, metrics.Requests.Success, metrics.Requests.Errors, - metrics.Requests.AvgLatency, metrics.Requests.P95Latency, metrics.Requests.P99Latency, - metrics.Requests.Throughput, metrics.Errors.Total, metrics.Errors.Rate, - metrics.Performance.ResponseTime, metrics.Performance.Throughput, - metrics.Performance.Concurrency, metrics.Performance.Saturation, metrics.Performance.Utilization, - metrics.Resources.CPUUsage, metrics.Resources.MemoryUsage, metrics.Resources.StorageUsage, - metrics.Resources.NetworkUsage, metrics.Resources.ResourceScore, - ) - + err := s.q.UpsertServiceMetrics(ctx, sqlcdb.UpsertServiceMetricsParams{ + ServiceID: metrics.ServiceID, + ServiceName: metrics.ServiceName, + ProjectID: metrics.ProjectID, + Timestamp: metrics.Timestamp, + RequestsTotal: int64N(metrics.Requests.Total), + RequestsSuccess: int64N(metrics.Requests.Success), + RequestsErrors: int64N(metrics.Requests.Errors), + RequestsAvgLatency: numStr(metrics.Requests.AvgLatency), + RequestsP95Latency: numStr(metrics.Requests.P95Latency), + RequestsP99Latency: numStr(metrics.Requests.P99Latency), + RequestsThroughput: numStr(metrics.Requests.Throughput), + ErrorsTotal: int64N(metrics.Errors.Total), + ErrorsRate: numStr(metrics.Errors.Rate), + PerformanceResponseTime: numStr(metrics.Performance.ResponseTime), + PerformanceThroughput: numStr(metrics.Performance.Throughput), + PerformanceConcurrency: int64N(metrics.Performance.Concurrency), + PerformanceSaturation: numStr(metrics.Performance.Saturation), + PerformanceUtilization: numStr(metrics.Performance.Utilization), + ResourceCpuUsage: numStr(metrics.Resources.CPUUsage), + ResourceMemoryUsage: int64N(metrics.Resources.MemoryUsage), + ResourceStorageUsage: int64N(metrics.Resources.StorageUsage), + ResourceNetworkUsage: int64N(metrics.Resources.NetworkUsage), + ResourceScore: numStr(metrics.Resources.ResourceScore), + }) if err != nil { return fmt.Errorf("failed to store service metrics: %w", err) } - // Store instance metrics + // The service row must land first — instance_metrics references + // service_metrics(service_id, timestamp). for _, instance := range metrics.Instances { if err := s.storeInstanceMetrics(ctx, metrics.ServiceID, metrics.Timestamp, instance); err != nil { return fmt.Errorf("failed to store instance metrics: %w", err) } } - return nil } // GetNodeMetrics retrieves node metrics from the database func (s *PostgreSQLMetricsStorage) GetNodeMetrics(ctx context.Context, nodeID string, from, to time.Time) ([]*NodeMetrics, error) { - query := ` - SELECT node_id, timestamp, cpu_usage, cpu_cores, load_avg_1, load_avg_5, load_avg_15, - memory_total, memory_used, memory_available, memory_usage_percent, - storage_total, storage_used, storage_available, storage_usage_percent, - network_bytes_in, network_bytes_out, network_packets_in, network_packets_out, - network_connections_in, network_connections_out, network_errors_in, network_errors_out, - uptime, processes, os, kernel, architecture - FROM node_metrics - WHERE node_id = $1 AND timestamp BETWEEN $2 AND $3 - ORDER BY timestamp ASC - ` - - rows, err := s.db.QueryContext(ctx, query, nodeID, from, to) + rows, err := s.q.ListNodeMetrics(ctx, sqlcdb.ListNodeMetricsParams{ + NodeID: nodeID, + Timestamp: from, + Timestamp_2: to, + }) if err != nil { return nil, fmt.Errorf("failed to query node metrics: %w", err) } - defer rows.Close() var metrics []*NodeMetrics - for rows.Next() { - var m NodeMetrics - err := rows.Scan( - &m.NodeID, &m.Timestamp, &m.CPU.UsagePercent, &m.CPU.UsageCores, - &m.CPU.LoadAverage1, &m.CPU.LoadAverage5, &m.CPU.LoadAverage15, - &m.Memory.Total, &m.Memory.Used, &m.Memory.Available, &m.Memory.UsagePercent, - &m.Storage.Total, &m.Storage.Used, &m.Storage.Available, &m.Storage.UsagePercent, - &m.Network.BytesIn, &m.Network.BytesOut, &m.Network.PacketsIn, &m.Network.PacketsOut, - &m.Network.ConnectionsIn, &m.Network.ConnectionsOut, &m.Network.ErrorsIn, &m.Network.ErrorsOut, - &m.System.Uptime, &m.System.Processes, &m.System.OS, &m.System.Kernel, &m.System.Architecture, - ) - if err != nil { - return nil, fmt.Errorf("failed to scan node metrics: %w", err) - } - - // Get container metrics for this timestamp - containers, err := s.getContainerMetrics(ctx, nodeID, m.Timestamp) - if err != nil { - return nil, fmt.Errorf("failed to get container metrics: %w", err) - } - m.Containers = containers - - metrics = append(metrics, &m) + for _, r := range rows { + metrics = append(metrics, &NodeMetrics{ + NodeID: r.NodeID, + Timestamp: r.Timestamp, + CPU: CPUMetrics{ + UsagePercent: f64v(r.CpuUsage), + UsageCores: f64v(r.CpuCores), + LoadAverage1: f64v(r.LoadAvg1), + LoadAverage5: f64v(r.LoadAvg5), + LoadAverage15: f64v(r.LoadAvg15), + }, + Memory: MemoryMetrics{ + Total: i64v(r.MemoryTotal), + Used: i64v(r.MemoryUsed), + Available: i64v(r.MemoryAvailable), + UsagePercent: f64v(r.MemoryUsagePercent), + }, + Storage: StorageMetrics{ + Total: i64v(r.StorageTotal), + Used: i64v(r.StorageUsed), + Available: i64v(r.StorageAvailable), + UsagePercent: f64v(r.StorageUsagePercent), + }, + Network: NetworkMetrics{ + BytesIn: i64v(r.NetworkBytesIn), + BytesOut: i64v(r.NetworkBytesOut), + PacketsIn: i64v(r.NetworkPacketsIn), + PacketsOut: i64v(r.NetworkPacketsOut), + ConnectionsIn: i32v(r.NetworkConnectionsIn), + ConnectionsOut: i32v(r.NetworkConnectionsOut), + ErrorsIn: i64v(r.NetworkErrorsIn), + ErrorsOut: i64v(r.NetworkErrorsOut), + }, + System: SystemMetrics{ + Uptime: time.Duration(r.UptimeSeconds) * time.Second, + Processes: int(r.Processes.Int32), + OS: r.Os.String, + Kernel: r.Kernel.String, + Architecture: r.Architecture.String, + }, + }) } - return metrics, nil } // GetServiceMetrics retrieves service metrics from the database func (s *PostgreSQLMetricsStorage) GetServiceMetrics(ctx context.Context, serviceID string, from, to time.Time) ([]*ServiceMetrics, error) { - query := ` - SELECT service_id, service_name, project_id, timestamp, - requests_total, requests_success, requests_errors, requests_avg_latency, - requests_p95_latency, requests_p99_latency, requests_throughput, - errors_total, errors_rate, performance_response_time, performance_throughput, - performance_concurrency, performance_saturation, performance_utilization, - resource_cpu_usage, resource_memory_usage, resource_storage_usage, - resource_network_usage, resource_score - FROM service_metrics - WHERE service_id = $1 AND timestamp BETWEEN $2 AND $3 - ORDER BY timestamp ASC - ` - - rows, err := s.db.QueryContext(ctx, query, serviceID, from, to) + rows, err := s.q.ListServiceMetrics(ctx, sqlcdb.ListServiceMetricsParams{ + ServiceID: serviceID, + Timestamp: from, + Timestamp_2: to, + }) if err != nil { return nil, fmt.Errorf("failed to query service metrics: %w", err) } - defer rows.Close() var metrics []*ServiceMetrics - for rows.Next() { - var m ServiceMetrics - err := rows.Scan( - &m.ServiceID, &m.ServiceName, &m.ProjectID, &m.Timestamp, - &m.Requests.Total, &m.Requests.Success, &m.Requests.Errors, - &m.Requests.AvgLatency, &m.Requests.P95Latency, &m.Requests.P99Latency, - &m.Requests.Throughput, &m.Errors.Total, &m.Errors.Rate, - &m.Performance.ResponseTime, &m.Performance.Throughput, - &m.Performance.Concurrency, &m.Performance.Saturation, &m.Performance.Utilization, - &m.Resources.CPUUsage, &m.Resources.MemoryUsage, &m.Resources.StorageUsage, - &m.Resources.NetworkUsage, &m.Resources.ResourceScore, - ) - if err != nil { - return nil, fmt.Errorf("failed to scan service metrics: %w", err) + for _, r := range rows { + m := &ServiceMetrics{ + ServiceID: r.ServiceID, + ServiceName: r.ServiceName, + ProjectID: r.ProjectID, + Timestamp: r.Timestamp, + Requests: RequestMetrics{ + Total: i64v(r.RequestsTotal), + Success: i64v(r.RequestsSuccess), + Errors: i64v(r.RequestsErrors), + AvgLatency: f64v(r.RequestsAvgLatency), + P95Latency: f64v(r.RequestsP95Latency), + P99Latency: f64v(r.RequestsP99Latency), + Throughput: f64v(r.RequestsThroughput), + }, + Errors: ErrorMetrics{ + Total: i64v(r.ErrorsTotal), + Rate: f64v(r.ErrorsRate), + }, + Performance: PerformanceMetrics{ + ResponseTime: f64v(r.PerformanceResponseTime), + Throughput: f64v(r.PerformanceThroughput), + Concurrency: i64v(r.PerformanceConcurrency), + Saturation: f64v(r.PerformanceSaturation), + Utilization: f64v(r.PerformanceUtilization), + }, + Resources: ResourceMetrics{ + CPUUsage: f64v(r.ResourceCpuUsage), + MemoryUsage: i64v(r.ResourceMemoryUsage), + StorageUsage: i64v(r.ResourceStorageUsage), + NetworkUsage: i64v(r.ResourceNetworkUsage), + ResourceScore: f64v(r.ResourceScore), + }, } - // Get instance metrics for this timestamp - instances, err := s.getInstanceMetrics(ctx, serviceID, m.Timestamp) + instances, err := s.getInstanceMetrics(ctx, serviceID, r.Timestamp) if err != nil { return nil, fmt.Errorf("failed to get instance metrics: %w", err) } m.Instances = instances - - metrics = append(metrics, &m) + metrics = append(metrics, m) } - return metrics, nil } -// GetAggregatedMetrics retrieves aggregated metrics based on a query +// GetAggregatedMetrics retrieves aggregated metrics based on a query. +// Bucketing uses date_bin (PostgreSQL 14+) — no TimescaleDB needed. func (s *PostgreSQLMetricsStorage) GetAggregatedMetrics(ctx context.Context, query MetricsQuery) (*AggregatedMetrics, error) { - // This is a simplified implementation - // In a real system, you'd build dynamic SQL based on the query - var timeSeries []TimeSeriesPoint var summary map[string]MetricSummary switch query.Type { case "node": - // Aggregate node metrics - nodeQuery := ` - SELECT - time_bucket($1, timestamp) AS bucket, - AVG(cpu_usage) as avg_cpu, - AVG(memory_usage_percent) as avg_memory, - AVG(storage_usage_percent) as avg_storage - FROM node_metrics - WHERE node_id = $2 AND timestamp BETWEEN $3 AND $4 - GROUP BY bucket - ORDER BY bucket ASC - ` - - rows, err := s.db.QueryContext(ctx, nodeQuery, query.Interval, query.ID, query.From, query.To) + seconds := int64(query.Interval.Seconds()) + if seconds <= 0 { + seconds = 300 + } + rows, err := s.q.AggregateNodeMetrics(ctx, sqlcdb.AggregateNodeMetricsParams{ + Column1: seconds, + NodeID: query.ID, + Timestamp: query.From, + Timestamp_2: query.To, + }) if err != nil { return nil, fmt.Errorf("failed to query aggregated node metrics: %w", err) } - defer rows.Close() - - for rows.Next() { - var bucket time.Time - var avgCPU, avgMemory, avgStorage float64 - if err := rows.Scan(&bucket, &avgCPU, &avgMemory, &avgStorage); err != nil { - return nil, fmt.Errorf("failed to scan aggregated metrics: %w", err) - } - - point := TimeSeriesPoint{ + for _, r := range rows { + bucket, _ := r.Bucket.(time.Time) + timeSeries = append(timeSeries, TimeSeriesPoint{ Timestamp: bucket, Values: map[string]float64{ - "cpu_usage": avgCPU, - "memory_usage": avgMemory, - "storage_usage": avgStorage, + "cpu_usage": r.AvgCpu, + "memory_usage": r.AvgMemory, + "storage_usage": r.AvgStorage, }, - } - timeSeries = append(timeSeries, point) + }) } - - // Calculate summary statistics summary = map[string]MetricSummary{ "cpu_usage": calculateSummary(timeSeries, "cpu_usage"), "memory_usage": calculateSummary(timeSeries, "memory_usage"), @@ -305,137 +298,69 @@ func (s *PostgreSQLMetricsStorage) GetAggregatedMetrics(ctx context.Context, que }, nil } -// Helper methods - -func (s *PostgreSQLMetricsStorage) storeContainerMetrics(ctx context.Context, nodeID string, timestamp time.Time, container ContainerMetrics) error { - query := ` - INSERT INTO container_metrics ( - node_id, timestamp, container_id, name, state, cpu, memory, - network_bytes_in, network_bytes_out, network_packets_in, network_packets_out, start_time - ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12) - ON CONFLICT (node_id, timestamp, container_id) DO UPDATE SET - name = EXCLUDED.name, - state = EXCLUDED.state, - cpu = EXCLUDED.cpu, - memory = EXCLUDED.memory, - network_bytes_in = EXCLUDED.network_bytes_in, - network_bytes_out = EXCLUDED.network_bytes_out, - network_packets_in = EXCLUDED.network_packets_in, - network_packets_out = EXCLUDED.network_packets_out, - start_time = EXCLUDED.start_time - ` - - _, err := s.db.ExecContext(ctx, query, - nodeID, timestamp, container.ContainerID, container.Name, container.State, - container.CPU, container.Memory, container.Network.BytesIn, container.Network.BytesOut, - container.Network.PacketsIn, container.Network.PacketsOut, container.StartTime, - ) - - return err -} - func (s *PostgreSQLMetricsStorage) storeInstanceMetrics(ctx context.Context, serviceID string, timestamp time.Time, instance InstanceMetrics) error { - query := ` - INSERT INTO instance_metrics ( - service_id, timestamp, instance_id, node_id, status, cpu, memory, - network_bytes_in, network_bytes_out, network_packets_in, network_packets_out, - network_connections_in, network_connections_out, network_errors_in, network_errors_out, - start_time, last_seen, health_status, health_last_check, health_check_count, health_failure_count - ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21) - ON CONFLICT (service_id, timestamp, instance_id) DO UPDATE SET - node_id = EXCLUDED.node_id, - status = EXCLUDED.status, - cpu = EXCLUDED.cpu, - memory = EXCLUDED.memory, - network_bytes_in = EXCLUDED.network_bytes_in, - network_bytes_out = EXCLUDED.network_bytes_out, - network_packets_in = EXCLUDED.network_packets_in, - network_packets_out = EXCLUDED.network_packets_out, - network_connections_in = EXCLUDED.network_connections_in, - network_connections_out = EXCLUDED.network_connections_out, - network_errors_in = EXCLUDED.network_errors_in, - network_errors_out = EXCLUDED.network_errors_out, - start_time = EXCLUDED.start_time, - last_seen = EXCLUDED.last_seen, - health_status = EXCLUDED.health_status, - health_last_check = EXCLUDED.health_last_check, - health_check_count = EXCLUDED.health_check_count, - health_failure_count = EXCLUDED.health_failure_count - ` - - _, err := s.db.ExecContext(ctx, query, - serviceID, timestamp, instance.InstanceID, instance.NodeID, instance.Status, - instance.CPU, instance.Memory, instance.Network.BytesIn, instance.Network.BytesOut, - instance.Network.PacketsIn, instance.Network.PacketsOut, instance.Network.ConnectionsIn, - instance.Network.ConnectionsOut, instance.Network.ErrorsIn, instance.Network.ErrorsOut, - instance.StartTime, instance.LastSeen, instance.Health.Status, instance.Health.LastCheck, - instance.Health.CheckCount, instance.Health.FailureCount, - ) - - return err -} - -func (s *PostgreSQLMetricsStorage) getContainerMetrics(ctx context.Context, nodeID string, timestamp time.Time) ([]ContainerMetrics, error) { - query := ` - SELECT container_id, name, state, cpu, memory, - network_bytes_in, network_bytes_out, network_packets_in, network_packets_out, start_time - FROM container_metrics - WHERE node_id = $1 AND timestamp = $2 - ` - - rows, err := s.db.QueryContext(ctx, query, nodeID, timestamp) - if err != nil { - return nil, err - } - defer rows.Close() - - var containers []ContainerMetrics - for rows.Next() { - var c ContainerMetrics - err := rows.Scan( - &c.ContainerID, &c.Name, &c.State, &c.CPU, &c.Memory, - &c.Network.BytesIn, &c.Network.BytesOut, &c.Network.PacketsIn, &c.Network.PacketsOut, &c.StartTime, - ) - if err != nil { - return nil, err - } - containers = append(containers, c) - } - - return containers, nil + return s.q.UpsertInstanceMetrics(ctx, sqlcdb.UpsertInstanceMetricsParams{ + ServiceID: serviceID, + Timestamp: timestamp, + InstanceID: instance.InstanceID, + NodeID: strN(instance.NodeID), + Status: strN(instance.Status), + Cpu: numStr(instance.CPU), + Memory: int64N(instance.Memory), + NetworkBytesIn: int64N(instance.Network.BytesIn), + NetworkBytesOut: int64N(instance.Network.BytesOut), + NetworkPacketsIn: int64N(instance.Network.PacketsIn), + NetworkPacketsOut: int64N(instance.Network.PacketsOut), + NetworkConnectionsIn: int32N(instance.Network.ConnectionsIn), + NetworkConnectionsOut: int32N(instance.Network.ConnectionsOut), + NetworkErrorsIn: int64N(instance.Network.ErrorsIn), + NetworkErrorsOut: int64N(instance.Network.ErrorsOut), + StartTime: timeN(instance.StartTime), + LastSeen: timeN(instance.LastSeen), + HealthStatus: strN(instance.Health.Status), + HealthLastCheck: timeN(instance.Health.LastCheck), + HealthCheckCount: sql.NullInt32{Int32: int32(instance.Health.CheckCount), Valid: true}, + HealthFailureCount: sql.NullInt32{Int32: int32(instance.Health.FailureCount), Valid: true}, + }) } func (s *PostgreSQLMetricsStorage) getInstanceMetrics(ctx context.Context, serviceID string, timestamp time.Time) ([]InstanceMetrics, error) { - query := ` - SELECT instance_id, node_id, status, cpu, memory, - network_bytes_in, network_bytes_out, network_packets_in, network_packets_out, - network_connections_in, network_connections_out, network_errors_in, network_errors_out, - start_time, last_seen, health_status, health_last_check, health_check_count, health_failure_count - FROM instance_metrics - WHERE service_id = $1 AND timestamp = $2 - ` - - rows, err := s.db.QueryContext(ctx, query, serviceID, timestamp) + rows, err := s.q.ListInstanceMetrics(ctx, sqlcdb.ListInstanceMetricsParams{ + ServiceID: serviceID, + Timestamp: timestamp, + }) if err != nil { return nil, err } - defer rows.Close() var instances []InstanceMetrics - for rows.Next() { - var i InstanceMetrics - err := rows.Scan( - &i.InstanceID, &i.NodeID, &i.Status, &i.CPU, &i.Memory, - &i.Network.BytesIn, &i.Network.BytesOut, &i.Network.PacketsIn, &i.Network.PacketsOut, - &i.Network.ConnectionsIn, &i.Network.ConnectionsOut, &i.Network.ErrorsIn, &i.Network.ErrorsOut, - &i.StartTime, &i.LastSeen, &i.Health.Status, &i.Health.LastCheck, &i.Health.CheckCount, &i.Health.FailureCount, - ) - if err != nil { - return nil, err - } - instances = append(instances, i) + for _, r := range rows { + instances = append(instances, InstanceMetrics{ + InstanceID: r.InstanceID, + NodeID: r.NodeID.String, + Status: r.Status.String, + CPU: f64v(r.Cpu), + Memory: i64v(r.Memory), + Network: NetworkMetrics{ + BytesIn: i64v(r.NetworkBytesIn), + BytesOut: i64v(r.NetworkBytesOut), + PacketsIn: i64v(r.NetworkPacketsIn), + PacketsOut: i64v(r.NetworkPacketsOut), + ConnectionsIn: i32v(r.NetworkConnectionsIn), + ConnectionsOut: i32v(r.NetworkConnectionsOut), + ErrorsIn: i64v(r.NetworkErrorsIn), + ErrorsOut: i64v(r.NetworkErrorsOut), + }, + StartTime: r.StartTime.Time, + LastSeen: r.LastSeen.Time, + Health: HealthMetrics{ + Status: r.HealthStatus.String, + LastCheck: r.HealthLastCheck.Time, + CheckCount: int(r.HealthCheckCount.Int32), + FailureCount: int(r.HealthFailureCount.Int32), + }, + }) } - return instances, nil } @@ -544,7 +469,9 @@ func (s *InMemoryMetricsStorage) GetServiceMetrics(ctx context.Context, serviceI } func (s *InMemoryMetricsStorage) GetAggregatedMetrics(ctx context.Context, query MetricsQuery) (*AggregatedMetrics, error) { - // Simplified implementation + s.mu.RLock() + defer s.mu.RUnlock() + return &AggregatedMetrics{ Query: query, TimeSeries: []TimeSeriesPoint{}, diff --git a/app/backend/internal/metrics/storage_test.go b/app/backend/internal/metrics/storage_test.go new file mode 100644 index 0000000..24f7e0a --- /dev/null +++ b/app/backend/internal/metrics/storage_test.go @@ -0,0 +1,130 @@ +package metrics + +import ( + "context" + "database/sql" + "os" + "testing" + "time" +) + +// Live round-trip against a real Postgres — set METRICS_TEST_DSN (or +// DATABASE_URL) to enable. Skipped by default so unit runs stay hermetic. +func testDSN(t *testing.T) *sql.DB { + t.Helper() + dsn := os.Getenv("METRICS_TEST_DSN") + if dsn == "" { + dsn = os.Getenv("DATABASE_URL") + } + if dsn == "" { + t.Skip("METRICS_TEST_DSN not set") + } + db, err := sql.Open("postgres", dsn) + if err != nil { + t.Fatalf("open: %v", err) + } + if err := db.PingContext(context.Background()); err != nil { + t.Fatalf("ping: %v", err) + } + t.Cleanup(func() { db.Close() }) + return db +} + +func TestPostgresStorageRoundTrip(t *testing.T) { + db := testDSN(t) + s := NewPostgreSQLMetricsStorage(db) + ctx := context.Background() + + ts := time.Now().UTC().Truncate(time.Second) + nodeID := "test-node-" + ts.Format("150405") + svcID := "test-svc-" + ts.Format("150405") + + // Clean slate + teardown + db.ExecContext(ctx, `DELETE FROM instance_metrics WHERE service_id = $1`, svcID) + db.ExecContext(ctx, `DELETE FROM service_metrics WHERE service_id = $1`, svcID) + db.ExecContext(ctx, `DELETE FROM node_metrics WHERE node_id = $1`, nodeID) + t.Cleanup(func() { + db.ExecContext(ctx, `DELETE FROM instance_metrics WHERE service_id = $1`, svcID) + db.ExecContext(ctx, `DELETE FROM service_metrics WHERE service_id = $1`, svcID) + db.ExecContext(ctx, `DELETE FROM node_metrics WHERE node_id = $1`, nodeID) + }) + + node := &NodeMetrics{ + NodeID: nodeID, + Timestamp: ts, + CPU: CPUMetrics{UsagePercent: 42.5, UsageCores: 1.7, LoadAverage1: 0.5, LoadAverage5: 0.4, LoadAverage15: 0.3}, + Memory: MemoryMetrics{Total: 16 << 30, Used: 8 << 30, Available: 8 << 30, UsagePercent: 50.0}, + Storage: StorageMetrics{Total: 100 << 30, Used: 40 << 30, Available: 60 << 30, UsagePercent: 40.0}, + Network: NetworkMetrics{BytesIn: 1000, BytesOut: 2000, PacketsIn: 10, PacketsOut: 20, ConnectionsIn: 3, ConnectionsOut: 4, ErrorsIn: 0, ErrorsOut: 1}, + System: SystemMetrics{Uptime: 3600 * time.Second, Processes: 120, OS: "linux", Kernel: "6.8", Architecture: "amd64"}, + } + if err := s.StoreNodeMetrics(ctx, node); err != nil { + t.Fatalf("store node: %v", err) + } + // Upsert idempotent on the same key. + if err := s.StoreNodeMetrics(ctx, node); err != nil { + t.Fatalf("store node (repeat): %v", err) + } + + svc := &ServiceMetrics{ + ServiceID: svcID, + ServiceName: "svc", + ProjectID: "proj", + Timestamp: ts, + Requests: RequestMetrics{Total: 100, Success: 95, Errors: 5, AvgLatency: 12.5, P95Latency: 80, P99Latency: 120, Throughput: 3.3}, + Errors: ErrorMetrics{Total: 5, Rate: 0.05}, + Performance: PerformanceMetrics{ResponseTime: 12.5, Throughput: 3.3, Concurrency: 2, Saturation: 0.4, Utilization: 0.55}, + Resources: ResourceMetrics{CPUUsage: 42.5, MemoryUsage: 512 << 20, StorageUsage: 1 << 30, NetworkUsage: 3000, ResourceScore: 0.7}, + Instances: []InstanceMetrics{{ + InstanceID: "inst-1", + NodeID: nodeID, + Status: "running", + CPU: 42.5, + Memory: 512 << 20, + Network: NetworkMetrics{BytesIn: 1000, BytesOut: 2000}, + StartTime: ts.Add(-time.Hour), + LastSeen: ts, + Health: HealthMetrics{Status: "healthy", LastCheck: ts, CheckCount: 10, FailureCount: 0}, + }}, + } + if err := s.StoreServiceMetrics(ctx, svc); err != nil { + t.Fatalf("store service: %v", err) + } + + from, to := ts.Add(-time.Minute), ts.Add(time.Minute) + gotNodes, err := s.GetNodeMetrics(ctx, nodeID, from, to) + if err != nil { + t.Fatalf("get nodes: %v", err) + } + if len(gotNodes) != 1 { + t.Fatalf("want 1 node row, got %d", len(gotNodes)) + } + if got := gotNodes[0]; got.CPU.UsagePercent != 42.5 || got.System.Processes != 120 || got.System.Uptime != time.Hour { + t.Errorf("node round-trip mismatch: %+v", got) + } + + gotSvcs, err := s.GetServiceMetrics(ctx, svcID, from, to) + if err != nil { + t.Fatalf("get services: %v", err) + } + if len(gotSvcs) != 1 { + t.Fatalf("want 1 service row, got %d", len(gotSvcs)) + } + if gotSvcs[0].Requests.P99Latency != 120 { + t.Errorf("p99 mismatch: %v", gotSvcs[0].Requests.P99Latency) + } + if len(gotSvcs[0].Instances) != 1 || gotSvcs[0].Instances[0].Health.Status != "healthy" { + t.Errorf("instance join mismatch: %+v", gotSvcs[0].Instances) + } + + agg, err := s.GetAggregatedMetrics(ctx, MetricsQuery{Type: "node", ID: nodeID, From: from, To: to, Interval: 5 * time.Minute}) + if err != nil { + t.Fatalf("aggregate: %v", err) + } + if len(agg.TimeSeries) != 1 || agg.TimeSeries[0].Values["cpu_usage"] != 42.5 { + t.Errorf("aggregate mismatch: %+v", agg.TimeSeries) + } + if agg.Summary["cpu_usage"].Avg != 42.5 { + t.Errorf("summary mismatch: %+v", agg.Summary) + } +} diff --git a/app/backend/sqlc/queries/domains.sql b/app/backend/sqlc/queries/domains.sql new file mode 100644 index 0000000..28f511e --- /dev/null +++ b/app/backend/sqlc/queries/domains.sql @@ -0,0 +1,46 @@ +-- name: ListServiceDomains :many +SELECT id, service_id, domain, is_default, cert_type, cert_status, last_checked_at, created_at +FROM service_domains +WHERE service_id = $1 +ORDER BY is_default DESC, created_at ASC; + +-- name: CountServiceDomains :one +SELECT COUNT(*) FROM service_domains WHERE service_id = $1; + +-- name: ClearServiceDomainDefault :exec +UPDATE service_domains SET is_default = false WHERE service_id = $1; + +-- name: CreateServiceDomain :one +INSERT INTO service_domains (service_id, domain, is_default) +VALUES ($1, $2, $3) +RETURNING id, service_id, domain, is_default, cert_type, cert_status, created_at; + +-- name: DeleteServiceDomain :execrows +DELETE FROM service_domains WHERE id = $1 AND service_id = $2; + +-- name: SetServiceDomainDefault :execrows +UPDATE service_domains SET is_default = (id = $1) WHERE service_id = $2; + +-- name: TouchServiceDomainChecked :exec +UPDATE service_domains SET last_checked_at = NOW() WHERE id = $1; + +-- name: GetServiceDomainDefault :one +SELECT domain FROM service_domains WHERE service_id = $1 AND is_default; + +-- name: GetServiceDomainFirst :one +SELECT domain FROM service_domains WHERE service_id = $1 ORDER BY created_at ASC LIMIT 1; + +-- name: SetServiceLegacyDomain :exec +UPDATE services SET domain = $1 WHERE id = $2; + +-- name: GetServicePlacementBrief :one +SELECT name, node_id, COALESCE(spread, false) AS spread FROM services WHERE id = $1; + +-- name: GetServiceAccess :one +SELECT maintenance_mode, basic_auth_users FROM services WHERE id = $1; + +-- name: ServiceWriteAccess :one +SELECT EXISTS( + SELECT 1 FROM services s JOIN projects p ON s.project_id = p.id + WHERE s.id = $1 AND (p.owner_id = $2 OR $3::bool) +) AS allowed; diff --git a/app/backend/sqlc/queries/metrics.sql b/app/backend/sqlc/queries/metrics.sql new file mode 100644 index 0000000..428d6b8 --- /dev/null +++ b/app/backend/sqlc/queries/metrics.sql @@ -0,0 +1,147 @@ +-- name: UpsertNodeMetrics :exec +INSERT INTO node_metrics ( + node_id, timestamp, cpu_usage, cpu_cores, load_avg_1, load_avg_5, load_avg_15, + memory_total, memory_used, memory_available, memory_usage_percent, + storage_total, storage_used, storage_available, storage_usage_percent, + network_bytes_in, network_bytes_out, network_packets_in, network_packets_out, + network_connections_in, network_connections_out, network_errors_in, network_errors_out, + uptime, processes, os, kernel, architecture +) VALUES ( + $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, + $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26, $27, $28 +) +ON CONFLICT (node_id, timestamp) DO UPDATE SET + cpu_usage = EXCLUDED.cpu_usage, + cpu_cores = EXCLUDED.cpu_cores, + load_avg_1 = EXCLUDED.load_avg_1, + load_avg_5 = EXCLUDED.load_avg_5, + load_avg_15 = EXCLUDED.load_avg_15, + memory_total = EXCLUDED.memory_total, + memory_used = EXCLUDED.memory_used, + memory_available = EXCLUDED.memory_available, + memory_usage_percent = EXCLUDED.memory_usage_percent, + storage_total = EXCLUDED.storage_total, + storage_used = EXCLUDED.storage_used, + storage_available = EXCLUDED.storage_available, + storage_usage_percent = EXCLUDED.storage_usage_percent, + network_bytes_in = EXCLUDED.network_bytes_in, + network_bytes_out = EXCLUDED.network_bytes_out, + network_packets_in = EXCLUDED.network_packets_in, + network_packets_out = EXCLUDED.network_packets_out, + network_connections_in = EXCLUDED.network_connections_in, + network_connections_out = EXCLUDED.network_connections_out, + network_errors_in = EXCLUDED.network_errors_in, + network_errors_out = EXCLUDED.network_errors_out, + uptime = EXCLUDED.uptime, + processes = EXCLUDED.processes, + os = EXCLUDED.os, + kernel = EXCLUDED.kernel, + architecture = EXCLUDED.architecture; + +-- name: UpsertServiceMetrics :exec +INSERT INTO service_metrics ( + service_id, service_name, project_id, timestamp, + requests_total, requests_success, requests_errors, requests_avg_latency, + requests_p95_latency, requests_p99_latency, requests_throughput, + errors_total, errors_rate, performance_response_time, performance_throughput, + performance_concurrency, performance_saturation, performance_utilization, + resource_cpu_usage, resource_memory_usage, resource_storage_usage, + resource_network_usage, resource_score +) VALUES ( + $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, + $16, $17, $18, $19, $20, $21, $22, $23 +) +ON CONFLICT (service_id, timestamp) DO UPDATE SET + requests_total = EXCLUDED.requests_total, + requests_success = EXCLUDED.requests_success, + requests_errors = EXCLUDED.requests_errors, + requests_avg_latency = EXCLUDED.requests_avg_latency, + requests_p95_latency = EXCLUDED.requests_p95_latency, + requests_p99_latency = EXCLUDED.requests_p99_latency, + requests_throughput = EXCLUDED.requests_throughput, + errors_total = EXCLUDED.errors_total, + errors_rate = EXCLUDED.errors_rate, + performance_response_time = EXCLUDED.performance_response_time, + performance_throughput = EXCLUDED.performance_throughput, + performance_concurrency = EXCLUDED.performance_concurrency, + performance_saturation = EXCLUDED.performance_saturation, + performance_utilization = EXCLUDED.performance_utilization, + resource_cpu_usage = EXCLUDED.resource_cpu_usage, + resource_memory_usage = EXCLUDED.resource_memory_usage, + resource_storage_usage = EXCLUDED.resource_storage_usage, + resource_network_usage = EXCLUDED.resource_network_usage, + resource_score = EXCLUDED.resource_score; + +-- name: UpsertInstanceMetrics :exec +INSERT INTO instance_metrics ( + service_id, timestamp, instance_id, node_id, status, cpu, memory, + network_bytes_in, network_bytes_out, network_packets_in, network_packets_out, + network_connections_in, network_connections_out, network_errors_in, network_errors_out, + start_time, last_seen, health_status, health_last_check, health_check_count, health_failure_count +) VALUES ( + $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, + $16, $17, $18, $19, $20, $21 +) +ON CONFLICT (service_id, timestamp, instance_id) DO UPDATE SET + node_id = EXCLUDED.node_id, + status = EXCLUDED.status, + cpu = EXCLUDED.cpu, + memory = EXCLUDED.memory, + network_bytes_in = EXCLUDED.network_bytes_in, + network_bytes_out = EXCLUDED.network_bytes_out, + network_packets_in = EXCLUDED.network_packets_in, + network_packets_out = EXCLUDED.network_packets_out, + network_connections_in = EXCLUDED.network_connections_in, + network_connections_out = EXCLUDED.network_connections_out, + network_errors_in = EXCLUDED.network_errors_in, + network_errors_out = EXCLUDED.network_errors_out, + start_time = EXCLUDED.start_time, + last_seen = EXCLUDED.last_seen, + health_status = EXCLUDED.health_status, + health_last_check = EXCLUDED.health_last_check, + health_check_count = EXCLUDED.health_check_count, + health_failure_count = EXCLUDED.health_failure_count; + +-- name: ListNodeMetrics :many +SELECT node_id, timestamp, cpu_usage, cpu_cores, load_avg_1, load_avg_5, load_avg_15, + memory_total, memory_used, memory_available, memory_usage_percent, + storage_total, storage_used, storage_available, storage_usage_percent, + network_bytes_in, network_bytes_out, network_packets_in, network_packets_out, + network_connections_in, network_connections_out, network_errors_in, network_errors_out, + coalesce(extract(epoch from uptime), 0)::bigint AS uptime_seconds, + processes, os, kernel, architecture +FROM node_metrics +WHERE node_id = $1 AND timestamp BETWEEN $2 AND $3 +ORDER BY timestamp ASC; + +-- name: ListServiceMetrics :many +SELECT service_id, service_name, project_id, timestamp, + requests_total, requests_success, requests_errors, requests_avg_latency, + requests_p95_latency, requests_p99_latency, requests_throughput, + errors_total, errors_rate, performance_response_time, performance_throughput, + performance_concurrency, performance_saturation, performance_utilization, + resource_cpu_usage, resource_memory_usage, resource_storage_usage, + resource_network_usage, resource_score +FROM service_metrics +WHERE service_id = $1 AND timestamp BETWEEN $2 AND $3 +ORDER BY timestamp ASC; + +-- name: ListInstanceMetrics :many +SELECT instance_id, node_id, status, cpu, memory, + network_bytes_in, network_bytes_out, network_packets_in, network_packets_out, + network_connections_in, network_connections_out, network_errors_in, network_errors_out, + start_time, last_seen, health_status, health_last_check, health_check_count, health_failure_count +FROM instance_metrics +WHERE service_id = $1 AND timestamp = $2; + +-- name: AggregateNodeMetrics :many +-- Portable bucket bucketing (no TimescaleDB): floor epoch seconds into +-- bucket-sized buckets, then back to timestamptz. +SELECT date_bin($1::interval, timestamp, '2000-01-01'::timestamptz) AS bucket, + AVG(cpu_usage)::float8 AS avg_cpu, + AVG(memory_usage_percent)::float8 AS avg_memory, + AVG(storage_usage_percent)::float8 AS avg_storage +FROM node_metrics +WHERE node_id = $2 AND timestamp BETWEEN $3 AND $4 +GROUP BY bucket +ORDER BY bucket ASC; diff --git a/app/backend/sqlc/schema.sql b/app/backend/sqlc/schema.sql index 10f0cc2..5fc2972 100644 --- a/app/backend/sqlc/schema.sql +++ b/app/backend/sqlc/schema.sql @@ -346,3 +346,90 @@ CREATE TABLE notification_channels ( created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(), updated_at TIMESTAMP WITH TIME ZONE DEFAULT NOW() ); + +CREATE TABLE node_metrics ( + node_id VARCHAR(255) NOT NULL, + timestamp TIMESTAMP WITH TIME ZONE NOT NULL, + cpu_usage NUMERIC(5,2), + cpu_cores NUMERIC(10,2), + load_avg_1 NUMERIC(5,2), + load_avg_5 NUMERIC(5,2), + load_avg_15 NUMERIC(5,2), + memory_total BIGINT, + memory_used BIGINT, + memory_available BIGINT, + memory_usage_percent NUMERIC(5,2), + storage_total BIGINT, + storage_used BIGINT, + storage_available BIGINT, + storage_usage_percent NUMERIC(5,2), + network_bytes_in BIGINT, + network_bytes_out BIGINT, + network_packets_in BIGINT, + network_packets_out BIGINT, + network_connections_in INTEGER, + network_connections_out INTEGER, + network_errors_in BIGINT, + network_errors_out BIGINT, + uptime INTERVAL, + processes INTEGER, + os VARCHAR(50), + kernel VARCHAR(50), + architecture VARCHAR(20), + created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(), + PRIMARY KEY (node_id, timestamp) +); + +CREATE TABLE service_metrics ( + service_id VARCHAR(255) NOT NULL, + service_name VARCHAR(255) NOT NULL, + project_id VARCHAR(255) NOT NULL, + timestamp TIMESTAMP WITH TIME ZONE NOT NULL, + requests_total BIGINT DEFAULT 0, + requests_success BIGINT DEFAULT 0, + requests_errors BIGINT DEFAULT 0, + requests_avg_latency NUMERIC(10,3), + requests_p95_latency NUMERIC(10,3), + requests_p99_latency NUMERIC(10,3), + requests_throughput NUMERIC(10,3), + errors_total BIGINT DEFAULT 0, + errors_rate NUMERIC(5,4), + performance_response_time NUMERIC(10,3), + performance_throughput NUMERIC(10,3), + performance_concurrency BIGINT, + performance_saturation NUMERIC(5,2), + performance_utilization NUMERIC(5,2), + resource_cpu_usage NUMERIC(5,2), + resource_memory_usage BIGINT, + resource_storage_usage BIGINT, + resource_network_usage BIGINT, + resource_score NUMERIC(5,2), + created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(), + PRIMARY KEY (service_id, timestamp) +); + +CREATE TABLE instance_metrics ( + service_id VARCHAR(255) NOT NULL, + timestamp TIMESTAMP WITH TIME ZONE NOT NULL, + instance_id VARCHAR(255) NOT NULL, + node_id VARCHAR(255), + status VARCHAR(50), + cpu NUMERIC(5,2), + memory BIGINT, + network_bytes_in BIGINT, + network_bytes_out BIGINT, + network_packets_in BIGINT, + network_packets_out BIGINT, + network_connections_in INTEGER, + network_connections_out INTEGER, + network_errors_in BIGINT, + network_errors_out BIGINT, + start_time TIMESTAMP WITH TIME ZONE, + last_seen TIMESTAMP WITH TIME ZONE, + health_status VARCHAR(20), + health_last_check TIMESTAMP WITH TIME ZONE, + health_check_count INTEGER DEFAULT 0, + health_failure_count INTEGER DEFAULT 0, + created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(), + PRIMARY KEY (service_id, timestamp, instance_id) +);