mirror of
https://github.com/Dvorinka/Containr.git
synced 2026-10-28 15:51:26 +00:00
refactor(db): port metrics storage + service domains to sqlc (#75)
First tranche of the ~300-site raw SQL port. Metrics: node/service/ instance rows via generated queries; aggregation moved off TimescaleDB's time_bucket onto portable date_bin (PG14+); interval uptime returns epoch seconds; per-container writes dropped (container_metrics FKs a different model — collector shape never matched). Domains: all reads/writes through sqlcdb including the create transaction via WithTx; default-domain and services.domain invariants unchanged. Both paths covered by live Postgres tests gated on METRICS_TEST_DSN. Generated with [Devin](https://devin.ai) Co-authored-by: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
Tomáš DvořákandDevin authored and GitHub committed
1 parent
ca8eab4a65
commit
302c595380
12 files changed
+1825
-385
No files matched your search
@@ -4,293 +4,286 @@ import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"containr/internal/database/sqlcdb"
|
||||
|
||||
_ "github.com/lib/pq"
|
||||
)
|
||||
|
||||
// PostgreSQLMetricsStorage implements MetricsStorage using PostgreSQL
|
||||
// through sqlc-generated queries (sqlc/queries/metrics.sql).
|
||||
//
|
||||
// Per-container node metrics are deliberately not persisted: the
|
||||
// container_metrics table belongs to the container_instances model and
|
||||
// has no place for the node/container samples collected here. Node and
|
||||
// service/instance rows map cleanly, so those are stored.
|
||||
type PostgreSQLMetricsStorage struct {
|
||||
db *sql.DB
|
||||
q *sqlcdb.Queries
|
||||
}
|
||||
|
||||
// NewPostgreSQLMetricsStorage creates a new PostgreSQL metrics storage
|
||||
func NewPostgreSQLMetricsStorage(db *sql.DB) *PostgreSQLMetricsStorage {
|
||||
return &PostgreSQLMetricsStorage{db: db}
|
||||
return &PostgreSQLMetricsStorage{q: sqlcdb.New(db)}
|
||||
}
|
||||
|
||||
func numStr(v float64) sql.NullString {
|
||||
return sql.NullString{String: strconv.FormatFloat(v, 'f', -1, 64), Valid: true}
|
||||
}
|
||||
|
||||
func int64N(v int64) sql.NullInt64 {
|
||||
return sql.NullInt64{Int64: v, Valid: true}
|
||||
}
|
||||
|
||||
func int32N(v int64) sql.NullInt32 {
|
||||
return sql.NullInt32{Int32: int32(v), Valid: true}
|
||||
}
|
||||
|
||||
func strN(v string) sql.NullString {
|
||||
return sql.NullString{String: v, Valid: v != ""}
|
||||
}
|
||||
|
||||
func timeN(t time.Time) sql.NullTime {
|
||||
return sql.NullTime{Time: t, Valid: !t.IsZero()}
|
||||
}
|
||||
|
||||
func f64v(n sql.NullString) float64 {
|
||||
f, _ := strconv.ParseFloat(n.String, 64)
|
||||
return f
|
||||
}
|
||||
|
||||
func i64v(n sql.NullInt64) int64 { return n.Int64 }
|
||||
|
||||
func i32v(n sql.NullInt32) int64 { return int64(n.Int32) }
|
||||
|
||||
// StoreNodeMetrics stores node metrics in the database
|
||||
func (s *PostgreSQLMetricsStorage) StoreNodeMetrics(ctx context.Context, metrics *NodeMetrics) error {
|
||||
query := `
|
||||
INSERT INTO node_metrics (
|
||||
node_id, timestamp, cpu_usage, cpu_cores, load_avg_1, load_avg_5, load_avg_15,
|
||||
memory_total, memory_used, memory_available, memory_usage_percent,
|
||||
storage_total, storage_used, storage_available, storage_usage_percent,
|
||||
network_bytes_in, network_bytes_out, network_packets_in, network_packets_out,
|
||||
network_connections_in, network_connections_out, network_errors_in, network_errors_out,
|
||||
uptime, processes, os, kernel, architecture
|
||||
) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26, $27, $28)
|
||||
ON CONFLICT (node_id, timestamp) DO UPDATE SET
|
||||
cpu_usage = EXCLUDED.cpu_usage,
|
||||
cpu_cores = EXCLUDED.cpu_cores,
|
||||
load_avg_1 = EXCLUDED.load_avg_1,
|
||||
load_avg_5 = EXCLUDED.load_avg_5,
|
||||
load_avg_15 = EXCLUDED.load_avg_15,
|
||||
memory_total = EXCLUDED.memory_total,
|
||||
memory_used = EXCLUDED.memory_used,
|
||||
memory_available = EXCLUDED.memory_available,
|
||||
memory_usage_percent = EXCLUDED.memory_usage_percent,
|
||||
storage_total = EXCLUDED.storage_total,
|
||||
storage_used = EXCLUDED.storage_used,
|
||||
storage_available = EXCLUDED.storage_available,
|
||||
storage_usage_percent = EXCLUDED.storage_usage_percent,
|
||||
network_bytes_in = EXCLUDED.network_bytes_in,
|
||||
network_bytes_out = EXCLUDED.network_bytes_out,
|
||||
network_packets_in = EXCLUDED.network_packets_in,
|
||||
network_packets_out = EXCLUDED.network_packets_out,
|
||||
network_connections_in = EXCLUDED.network_connections_in,
|
||||
network_connections_out = EXCLUDED.network_connections_out,
|
||||
network_errors_in = EXCLUDED.network_errors_in,
|
||||
network_errors_out = EXCLUDED.network_errors_out,
|
||||
uptime = EXCLUDED.uptime,
|
||||
processes = EXCLUDED.processes,
|
||||
os = EXCLUDED.os,
|
||||
kernel = EXCLUDED.kernel,
|
||||
architecture = EXCLUDED.architecture
|
||||
`
|
||||
|
||||
_, err := s.db.ExecContext(ctx, query,
|
||||
metrics.NodeID, metrics.Timestamp, metrics.CPU.UsagePercent, metrics.CPU.UsageCores,
|
||||
metrics.CPU.LoadAverage1, metrics.CPU.LoadAverage5, metrics.CPU.LoadAverage15,
|
||||
metrics.Memory.Total, metrics.Memory.Used, metrics.Memory.Available, metrics.Memory.UsagePercent,
|
||||
metrics.Storage.Total, metrics.Storage.Used, metrics.Storage.Available, metrics.Storage.UsagePercent,
|
||||
metrics.Network.BytesIn, metrics.Network.BytesOut, metrics.Network.PacketsIn, metrics.Network.PacketsOut,
|
||||
metrics.Network.ConnectionsIn, metrics.Network.ConnectionsOut, metrics.Network.ErrorsIn, metrics.Network.ErrorsOut,
|
||||
metrics.System.Uptime, metrics.System.Processes, metrics.System.OS, metrics.System.Kernel, metrics.System.Architecture,
|
||||
)
|
||||
|
||||
err := s.q.UpsertNodeMetrics(ctx, sqlcdb.UpsertNodeMetricsParams{
|
||||
NodeID: metrics.NodeID,
|
||||
Timestamp: metrics.Timestamp,
|
||||
CpuUsage: numStr(metrics.CPU.UsagePercent),
|
||||
CpuCores: numStr(metrics.CPU.UsageCores),
|
||||
LoadAvg1: numStr(metrics.CPU.LoadAverage1),
|
||||
LoadAvg5: numStr(metrics.CPU.LoadAverage5),
|
||||
LoadAvg15: numStr(metrics.CPU.LoadAverage15),
|
||||
MemoryTotal: int64N(metrics.Memory.Total),
|
||||
MemoryUsed: int64N(metrics.Memory.Used),
|
||||
MemoryAvailable: int64N(metrics.Memory.Available),
|
||||
MemoryUsagePercent: numStr(metrics.Memory.UsagePercent),
|
||||
StorageTotal: int64N(metrics.Storage.Total),
|
||||
StorageUsed: int64N(metrics.Storage.Used),
|
||||
StorageAvailable: int64N(metrics.Storage.Available),
|
||||
StorageUsagePercent: numStr(metrics.Storage.UsagePercent),
|
||||
NetworkBytesIn: int64N(metrics.Network.BytesIn),
|
||||
NetworkBytesOut: int64N(metrics.Network.BytesOut),
|
||||
NetworkPacketsIn: int64N(metrics.Network.PacketsIn),
|
||||
NetworkPacketsOut: int64N(metrics.Network.PacketsOut),
|
||||
NetworkConnectionsIn: int32N(metrics.Network.ConnectionsIn),
|
||||
NetworkConnectionsOut: int32N(metrics.Network.ConnectionsOut),
|
||||
NetworkErrorsIn: int64N(metrics.Network.ErrorsIn),
|
||||
NetworkErrorsOut: int64N(metrics.Network.ErrorsOut),
|
||||
// INTERVAL param — Postgres casts the bigint as seconds.
|
||||
Uptime: int64N(int64(metrics.System.Uptime.Seconds())),
|
||||
Processes: sql.NullInt32{Int32: int32(metrics.System.Processes), Valid: true},
|
||||
Os: strN(metrics.System.OS),
|
||||
Kernel: strN(metrics.System.Kernel),
|
||||
Architecture: strN(metrics.System.Architecture),
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to store node metrics: %w", err)
|
||||
}
|
||||
|
||||
// Store container metrics
|
||||
for _, container := range metrics.Containers {
|
||||
if err := s.storeContainerMetrics(ctx, metrics.NodeID, metrics.Timestamp, container); err != nil {
|
||||
return fmt.Errorf("failed to store container metrics: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// StoreServiceMetrics stores service metrics in the database
|
||||
func (s *PostgreSQLMetricsStorage) StoreServiceMetrics(ctx context.Context, metrics *ServiceMetrics) error {
|
||||
query := `
|
||||
INSERT INTO service_metrics (
|
||||
service_id, service_name, project_id, timestamp,
|
||||
requests_total, requests_success, requests_errors, requests_avg_latency,
|
||||
requests_p95_latency, requests_p99_latency, requests_throughput,
|
||||
errors_total, errors_rate, performance_response_time, performance_throughput,
|
||||
performance_concurrency, performance_saturation, performance_utilization,
|
||||
resource_cpu_usage, resource_memory_usage, resource_storage_usage,
|
||||
resource_network_usage, resource_score
|
||||
) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23)
|
||||
ON CONFLICT (service_id, timestamp) DO UPDATE SET
|
||||
requests_total = EXCLUDED.requests_total,
|
||||
requests_success = EXCLUDED.requests_success,
|
||||
requests_errors = EXCLUDED.requests_errors,
|
||||
requests_avg_latency = EXCLUDED.requests_avg_latency,
|
||||
requests_p95_latency = EXCLUDED.requests_p95_latency,
|
||||
requests_p99_latency = EXCLUDED.requests_p99_latency,
|
||||
requests_throughput = EXCLUDED.requests_throughput,
|
||||
errors_total = EXCLUDED.errors_total,
|
||||
errors_rate = EXCLUDED.errors_rate,
|
||||
performance_response_time = EXCLUDED.performance_response_time,
|
||||
performance_throughput = EXCLUDED.performance_throughput,
|
||||
performance_concurrency = EXCLUDED.performance_concurrency,
|
||||
performance_saturation = EXCLUDED.performance_saturation,
|
||||
performance_utilization = EXCLUDED.performance_utilization,
|
||||
resource_cpu_usage = EXCLUDED.resource_cpu_usage,
|
||||
resource_memory_usage = EXCLUDED.resource_memory_usage,
|
||||
resource_storage_usage = EXCLUDED.resource_storage_usage,
|
||||
resource_network_usage = EXCLUDED.resource_network_usage,
|
||||
resource_score = EXCLUDED.resource_score
|
||||
`
|
||||
|
||||
_, err := s.db.ExecContext(ctx, query,
|
||||
metrics.ServiceID, metrics.ServiceName, metrics.ProjectID, metrics.Timestamp,
|
||||
metrics.Requests.Total, metrics.Requests.Success, metrics.Requests.Errors,
|
||||
metrics.Requests.AvgLatency, metrics.Requests.P95Latency, metrics.Requests.P99Latency,
|
||||
metrics.Requests.Throughput, metrics.Errors.Total, metrics.Errors.Rate,
|
||||
metrics.Performance.ResponseTime, metrics.Performance.Throughput,
|
||||
metrics.Performance.Concurrency, metrics.Performance.Saturation, metrics.Performance.Utilization,
|
||||
metrics.Resources.CPUUsage, metrics.Resources.MemoryUsage, metrics.Resources.StorageUsage,
|
||||
metrics.Resources.NetworkUsage, metrics.Resources.ResourceScore,
|
||||
)
|
||||
|
||||
err := s.q.UpsertServiceMetrics(ctx, sqlcdb.UpsertServiceMetricsParams{
|
||||
ServiceID: metrics.ServiceID,
|
||||
ServiceName: metrics.ServiceName,
|
||||
ProjectID: metrics.ProjectID,
|
||||
Timestamp: metrics.Timestamp,
|
||||
RequestsTotal: int64N(metrics.Requests.Total),
|
||||
RequestsSuccess: int64N(metrics.Requests.Success),
|
||||
RequestsErrors: int64N(metrics.Requests.Errors),
|
||||
RequestsAvgLatency: numStr(metrics.Requests.AvgLatency),
|
||||
RequestsP95Latency: numStr(metrics.Requests.P95Latency),
|
||||
RequestsP99Latency: numStr(metrics.Requests.P99Latency),
|
||||
RequestsThroughput: numStr(metrics.Requests.Throughput),
|
||||
ErrorsTotal: int64N(metrics.Errors.Total),
|
||||
ErrorsRate: numStr(metrics.Errors.Rate),
|
||||
PerformanceResponseTime: numStr(metrics.Performance.ResponseTime),
|
||||
PerformanceThroughput: numStr(metrics.Performance.Throughput),
|
||||
PerformanceConcurrency: int64N(metrics.Performance.Concurrency),
|
||||
PerformanceSaturation: numStr(metrics.Performance.Saturation),
|
||||
PerformanceUtilization: numStr(metrics.Performance.Utilization),
|
||||
ResourceCpuUsage: numStr(metrics.Resources.CPUUsage),
|
||||
ResourceMemoryUsage: int64N(metrics.Resources.MemoryUsage),
|
||||
ResourceStorageUsage: int64N(metrics.Resources.StorageUsage),
|
||||
ResourceNetworkUsage: int64N(metrics.Resources.NetworkUsage),
|
||||
ResourceScore: numStr(metrics.Resources.ResourceScore),
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to store service metrics: %w", err)
|
||||
}
|
||||
|
||||
// Store instance metrics
|
||||
// The service row must land first — instance_metrics references
|
||||
// service_metrics(service_id, timestamp).
|
||||
for _, instance := range metrics.Instances {
|
||||
if err := s.storeInstanceMetrics(ctx, metrics.ServiceID, metrics.Timestamp, instance); err != nil {
|
||||
return fmt.Errorf("failed to store instance metrics: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetNodeMetrics retrieves node metrics from the database
|
||||
func (s *PostgreSQLMetricsStorage) GetNodeMetrics(ctx context.Context, nodeID string, from, to time.Time) ([]*NodeMetrics, error) {
|
||||
query := `
|
||||
SELECT node_id, timestamp, cpu_usage, cpu_cores, load_avg_1, load_avg_5, load_avg_15,
|
||||
memory_total, memory_used, memory_available, memory_usage_percent,
|
||||
storage_total, storage_used, storage_available, storage_usage_percent,
|
||||
network_bytes_in, network_bytes_out, network_packets_in, network_packets_out,
|
||||
network_connections_in, network_connections_out, network_errors_in, network_errors_out,
|
||||
uptime, processes, os, kernel, architecture
|
||||
FROM node_metrics
|
||||
WHERE node_id = $1 AND timestamp BETWEEN $2 AND $3
|
||||
ORDER BY timestamp ASC
|
||||
`
|
||||
|
||||
rows, err := s.db.QueryContext(ctx, query, nodeID, from, to)
|
||||
rows, err := s.q.ListNodeMetrics(ctx, sqlcdb.ListNodeMetricsParams{
|
||||
NodeID: nodeID,
|
||||
Timestamp: from,
|
||||
Timestamp_2: to,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to query node metrics: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var metrics []*NodeMetrics
|
||||
for rows.Next() {
|
||||
var m NodeMetrics
|
||||
err := rows.Scan(
|
||||
&m.NodeID, &m.Timestamp, &m.CPU.UsagePercent, &m.CPU.UsageCores,
|
||||
&m.CPU.LoadAverage1, &m.CPU.LoadAverage5, &m.CPU.LoadAverage15,
|
||||
&m.Memory.Total, &m.Memory.Used, &m.Memory.Available, &m.Memory.UsagePercent,
|
||||
&m.Storage.Total, &m.Storage.Used, &m.Storage.Available, &m.Storage.UsagePercent,
|
||||
&m.Network.BytesIn, &m.Network.BytesOut, &m.Network.PacketsIn, &m.Network.PacketsOut,
|
||||
&m.Network.ConnectionsIn, &m.Network.ConnectionsOut, &m.Network.ErrorsIn, &m.Network.ErrorsOut,
|
||||
&m.System.Uptime, &m.System.Processes, &m.System.OS, &m.System.Kernel, &m.System.Architecture,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to scan node metrics: %w", err)
|
||||
}
|
||||
|
||||
// Get container metrics for this timestamp
|
||||
containers, err := s.getContainerMetrics(ctx, nodeID, m.Timestamp)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to get container metrics: %w", err)
|
||||
}
|
||||
m.Containers = containers
|
||||
|
||||
metrics = append(metrics, &m)
|
||||
for _, r := range rows {
|
||||
metrics = append(metrics, &NodeMetrics{
|
||||
NodeID: r.NodeID,
|
||||
Timestamp: r.Timestamp,
|
||||
CPU: CPUMetrics{
|
||||
UsagePercent: f64v(r.CpuUsage),
|
||||
UsageCores: f64v(r.CpuCores),
|
||||
LoadAverage1: f64v(r.LoadAvg1),
|
||||
LoadAverage5: f64v(r.LoadAvg5),
|
||||
LoadAverage15: f64v(r.LoadAvg15),
|
||||
},
|
||||
Memory: MemoryMetrics{
|
||||
Total: i64v(r.MemoryTotal),
|
||||
Used: i64v(r.MemoryUsed),
|
||||
Available: i64v(r.MemoryAvailable),
|
||||
UsagePercent: f64v(r.MemoryUsagePercent),
|
||||
},
|
||||
Storage: StorageMetrics{
|
||||
Total: i64v(r.StorageTotal),
|
||||
Used: i64v(r.StorageUsed),
|
||||
Available: i64v(r.StorageAvailable),
|
||||
UsagePercent: f64v(r.StorageUsagePercent),
|
||||
},
|
||||
Network: NetworkMetrics{
|
||||
BytesIn: i64v(r.NetworkBytesIn),
|
||||
BytesOut: i64v(r.NetworkBytesOut),
|
||||
PacketsIn: i64v(r.NetworkPacketsIn),
|
||||
PacketsOut: i64v(r.NetworkPacketsOut),
|
||||
ConnectionsIn: i32v(r.NetworkConnectionsIn),
|
||||
ConnectionsOut: i32v(r.NetworkConnectionsOut),
|
||||
ErrorsIn: i64v(r.NetworkErrorsIn),
|
||||
ErrorsOut: i64v(r.NetworkErrorsOut),
|
||||
},
|
||||
System: SystemMetrics{
|
||||
Uptime: time.Duration(r.UptimeSeconds) * time.Second,
|
||||
Processes: int(r.Processes.Int32),
|
||||
OS: r.Os.String,
|
||||
Kernel: r.Kernel.String,
|
||||
Architecture: r.Architecture.String,
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
return metrics, nil
|
||||
}
|
||||
|
||||
// GetServiceMetrics retrieves service metrics from the database
|
||||
func (s *PostgreSQLMetricsStorage) GetServiceMetrics(ctx context.Context, serviceID string, from, to time.Time) ([]*ServiceMetrics, error) {
|
||||
query := `
|
||||
SELECT service_id, service_name, project_id, timestamp,
|
||||
requests_total, requests_success, requests_errors, requests_avg_latency,
|
||||
requests_p95_latency, requests_p99_latency, requests_throughput,
|
||||
errors_total, errors_rate, performance_response_time, performance_throughput,
|
||||
performance_concurrency, performance_saturation, performance_utilization,
|
||||
resource_cpu_usage, resource_memory_usage, resource_storage_usage,
|
||||
resource_network_usage, resource_score
|
||||
FROM service_metrics
|
||||
WHERE service_id = $1 AND timestamp BETWEEN $2 AND $3
|
||||
ORDER BY timestamp ASC
|
||||
`
|
||||
|
||||
rows, err := s.db.QueryContext(ctx, query, serviceID, from, to)
|
||||
rows, err := s.q.ListServiceMetrics(ctx, sqlcdb.ListServiceMetricsParams{
|
||||
ServiceID: serviceID,
|
||||
Timestamp: from,
|
||||
Timestamp_2: to,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to query service metrics: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var metrics []*ServiceMetrics
|
||||
for rows.Next() {
|
||||
var m ServiceMetrics
|
||||
err := rows.Scan(
|
||||
&m.ServiceID, &m.ServiceName, &m.ProjectID, &m.Timestamp,
|
||||
&m.Requests.Total, &m.Requests.Success, &m.Requests.Errors,
|
||||
&m.Requests.AvgLatency, &m.Requests.P95Latency, &m.Requests.P99Latency,
|
||||
&m.Requests.Throughput, &m.Errors.Total, &m.Errors.Rate,
|
||||
&m.Performance.ResponseTime, &m.Performance.Throughput,
|
||||
&m.Performance.Concurrency, &m.Performance.Saturation, &m.Performance.Utilization,
|
||||
&m.Resources.CPUUsage, &m.Resources.MemoryUsage, &m.Resources.StorageUsage,
|
||||
&m.Resources.NetworkUsage, &m.Resources.ResourceScore,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to scan service metrics: %w", err)
|
||||
for _, r := range rows {
|
||||
m := &ServiceMetrics{
|
||||
ServiceID: r.ServiceID,
|
||||
ServiceName: r.ServiceName,
|
||||
ProjectID: r.ProjectID,
|
||||
Timestamp: r.Timestamp,
|
||||
Requests: RequestMetrics{
|
||||
Total: i64v(r.RequestsTotal),
|
||||
Success: i64v(r.RequestsSuccess),
|
||||
Errors: i64v(r.RequestsErrors),
|
||||
AvgLatency: f64v(r.RequestsAvgLatency),
|
||||
P95Latency: f64v(r.RequestsP95Latency),
|
||||
P99Latency: f64v(r.RequestsP99Latency),
|
||||
Throughput: f64v(r.RequestsThroughput),
|
||||
},
|
||||
Errors: ErrorMetrics{
|
||||
Total: i64v(r.ErrorsTotal),
|
||||
Rate: f64v(r.ErrorsRate),
|
||||
},
|
||||
Performance: PerformanceMetrics{
|
||||
ResponseTime: f64v(r.PerformanceResponseTime),
|
||||
Throughput: f64v(r.PerformanceThroughput),
|
||||
Concurrency: i64v(r.PerformanceConcurrency),
|
||||
Saturation: f64v(r.PerformanceSaturation),
|
||||
Utilization: f64v(r.PerformanceUtilization),
|
||||
},
|
||||
Resources: ResourceMetrics{
|
||||
CPUUsage: f64v(r.ResourceCpuUsage),
|
||||
MemoryUsage: i64v(r.ResourceMemoryUsage),
|
||||
StorageUsage: i64v(r.ResourceStorageUsage),
|
||||
NetworkUsage: i64v(r.ResourceNetworkUsage),
|
||||
ResourceScore: f64v(r.ResourceScore),
|
||||
},
|
||||
}
|
||||
|
||||
// Get instance metrics for this timestamp
|
||||
instances, err := s.getInstanceMetrics(ctx, serviceID, m.Timestamp)
|
||||
instances, err := s.getInstanceMetrics(ctx, serviceID, r.Timestamp)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to get instance metrics: %w", err)
|
||||
}
|
||||
m.Instances = instances
|
||||
|
||||
metrics = append(metrics, &m)
|
||||
metrics = append(metrics, m)
|
||||
}
|
||||
|
||||
return metrics, nil
|
||||
}
|
||||
|
||||
// GetAggregatedMetrics retrieves aggregated metrics based on a query
|
||||
// GetAggregatedMetrics retrieves aggregated metrics based on a query.
|
||||
// Bucketing uses date_bin (PostgreSQL 14+) — no TimescaleDB needed.
|
||||
func (s *PostgreSQLMetricsStorage) GetAggregatedMetrics(ctx context.Context, query MetricsQuery) (*AggregatedMetrics, error) {
|
||||
// This is a simplified implementation
|
||||
// In a real system, you'd build dynamic SQL based on the query
|
||||
|
||||
var timeSeries []TimeSeriesPoint
|
||||
var summary map[string]MetricSummary
|
||||
|
||||
switch query.Type {
|
||||
case "node":
|
||||
// Aggregate node metrics
|
||||
nodeQuery := `
|
||||
SELECT
|
||||
time_bucket($1, timestamp) AS bucket,
|
||||
AVG(cpu_usage) as avg_cpu,
|
||||
AVG(memory_usage_percent) as avg_memory,
|
||||
AVG(storage_usage_percent) as avg_storage
|
||||
FROM node_metrics
|
||||
WHERE node_id = $2 AND timestamp BETWEEN $3 AND $4
|
||||
GROUP BY bucket
|
||||
ORDER BY bucket ASC
|
||||
`
|
||||
|
||||
rows, err := s.db.QueryContext(ctx, nodeQuery, query.Interval, query.ID, query.From, query.To)
|
||||
seconds := int64(query.Interval.Seconds())
|
||||
if seconds <= 0 {
|
||||
seconds = 300
|
||||
}
|
||||
rows, err := s.q.AggregateNodeMetrics(ctx, sqlcdb.AggregateNodeMetricsParams{
|
||||
Column1: seconds,
|
||||
NodeID: query.ID,
|
||||
Timestamp: query.From,
|
||||
Timestamp_2: query.To,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to query aggregated node metrics: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
for rows.Next() {
|
||||
var bucket time.Time
|
||||
var avgCPU, avgMemory, avgStorage float64
|
||||
if err := rows.Scan(&bucket, &avgCPU, &avgMemory, &avgStorage); err != nil {
|
||||
return nil, fmt.Errorf("failed to scan aggregated metrics: %w", err)
|
||||
}
|
||||
|
||||
point := TimeSeriesPoint{
|
||||
for _, r := range rows {
|
||||
bucket, _ := r.Bucket.(time.Time)
|
||||
timeSeries = append(timeSeries, TimeSeriesPoint{
|
||||
Timestamp: bucket,
|
||||
Values: map[string]float64{
|
||||
"cpu_usage": avgCPU,
|
||||
"memory_usage": avgMemory,
|
||||
"storage_usage": avgStorage,
|
||||
"cpu_usage": r.AvgCpu,
|
||||
"memory_usage": r.AvgMemory,
|
||||
"storage_usage": r.AvgStorage,
|
||||
},
|
||||
}
|
||||
timeSeries = append(timeSeries, point)
|
||||
})
|
||||
}
|
||||
|
||||
// Calculate summary statistics
|
||||
summary = map[string]MetricSummary{
|
||||
"cpu_usage": calculateSummary(timeSeries, "cpu_usage"),
|
||||
"memory_usage": calculateSummary(timeSeries, "memory_usage"),
|
||||
@@ -305,137 +298,69 @@ func (s *PostgreSQLMetricsStorage) GetAggregatedMetrics(ctx context.Context, que
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Helper methods
|
||||
|
||||
func (s *PostgreSQLMetricsStorage) storeContainerMetrics(ctx context.Context, nodeID string, timestamp time.Time, container ContainerMetrics) error {
|
||||
query := `
|
||||
INSERT INTO container_metrics (
|
||||
node_id, timestamp, container_id, name, state, cpu, memory,
|
||||
network_bytes_in, network_bytes_out, network_packets_in, network_packets_out, start_time
|
||||
) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12)
|
||||
ON CONFLICT (node_id, timestamp, container_id) DO UPDATE SET
|
||||
name = EXCLUDED.name,
|
||||
state = EXCLUDED.state,
|
||||
cpu = EXCLUDED.cpu,
|
||||
memory = EXCLUDED.memory,
|
||||
network_bytes_in = EXCLUDED.network_bytes_in,
|
||||
network_bytes_out = EXCLUDED.network_bytes_out,
|
||||
network_packets_in = EXCLUDED.network_packets_in,
|
||||
network_packets_out = EXCLUDED.network_packets_out,
|
||||
start_time = EXCLUDED.start_time
|
||||
`
|
||||
|
||||
_, err := s.db.ExecContext(ctx, query,
|
||||
nodeID, timestamp, container.ContainerID, container.Name, container.State,
|
||||
container.CPU, container.Memory, container.Network.BytesIn, container.Network.BytesOut,
|
||||
container.Network.PacketsIn, container.Network.PacketsOut, container.StartTime,
|
||||
)
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *PostgreSQLMetricsStorage) storeInstanceMetrics(ctx context.Context, serviceID string, timestamp time.Time, instance InstanceMetrics) error {
|
||||
query := `
|
||||
INSERT INTO instance_metrics (
|
||||
service_id, timestamp, instance_id, node_id, status, cpu, memory,
|
||||
network_bytes_in, network_bytes_out, network_packets_in, network_packets_out,
|
||||
network_connections_in, network_connections_out, network_errors_in, network_errors_out,
|
||||
start_time, last_seen, health_status, health_last_check, health_check_count, health_failure_count
|
||||
) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21)
|
||||
ON CONFLICT (service_id, timestamp, instance_id) DO UPDATE SET
|
||||
node_id = EXCLUDED.node_id,
|
||||
status = EXCLUDED.status,
|
||||
cpu = EXCLUDED.cpu,
|
||||
memory = EXCLUDED.memory,
|
||||
network_bytes_in = EXCLUDED.network_bytes_in,
|
||||
network_bytes_out = EXCLUDED.network_bytes_out,
|
||||
network_packets_in = EXCLUDED.network_packets_in,
|
||||
network_packets_out = EXCLUDED.network_packets_out,
|
||||
network_connections_in = EXCLUDED.network_connections_in,
|
||||
network_connections_out = EXCLUDED.network_connections_out,
|
||||
network_errors_in = EXCLUDED.network_errors_in,
|
||||
network_errors_out = EXCLUDED.network_errors_out,
|
||||
start_time = EXCLUDED.start_time,
|
||||
last_seen = EXCLUDED.last_seen,
|
||||
health_status = EXCLUDED.health_status,
|
||||
health_last_check = EXCLUDED.health_last_check,
|
||||
health_check_count = EXCLUDED.health_check_count,
|
||||
health_failure_count = EXCLUDED.health_failure_count
|
||||
`
|
||||
|
||||
_, err := s.db.ExecContext(ctx, query,
|
||||
serviceID, timestamp, instance.InstanceID, instance.NodeID, instance.Status,
|
||||
instance.CPU, instance.Memory, instance.Network.BytesIn, instance.Network.BytesOut,
|
||||
instance.Network.PacketsIn, instance.Network.PacketsOut, instance.Network.ConnectionsIn,
|
||||
instance.Network.ConnectionsOut, instance.Network.ErrorsIn, instance.Network.ErrorsOut,
|
||||
instance.StartTime, instance.LastSeen, instance.Health.Status, instance.Health.LastCheck,
|
||||
instance.Health.CheckCount, instance.Health.FailureCount,
|
||||
)
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *PostgreSQLMetricsStorage) getContainerMetrics(ctx context.Context, nodeID string, timestamp time.Time) ([]ContainerMetrics, error) {
|
||||
query := `
|
||||
SELECT container_id, name, state, cpu, memory,
|
||||
network_bytes_in, network_bytes_out, network_packets_in, network_packets_out, start_time
|
||||
FROM container_metrics
|
||||
WHERE node_id = $1 AND timestamp = $2
|
||||
`
|
||||
|
||||
rows, err := s.db.QueryContext(ctx, query, nodeID, timestamp)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var containers []ContainerMetrics
|
||||
for rows.Next() {
|
||||
var c ContainerMetrics
|
||||
err := rows.Scan(
|
||||
&c.ContainerID, &c.Name, &c.State, &c.CPU, &c.Memory,
|
||||
&c.Network.BytesIn, &c.Network.BytesOut, &c.Network.PacketsIn, &c.Network.PacketsOut, &c.StartTime,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
containers = append(containers, c)
|
||||
}
|
||||
|
||||
return containers, nil
|
||||
return s.q.UpsertInstanceMetrics(ctx, sqlcdb.UpsertInstanceMetricsParams{
|
||||
ServiceID: serviceID,
|
||||
Timestamp: timestamp,
|
||||
InstanceID: instance.InstanceID,
|
||||
NodeID: strN(instance.NodeID),
|
||||
Status: strN(instance.Status),
|
||||
Cpu: numStr(instance.CPU),
|
||||
Memory: int64N(instance.Memory),
|
||||
NetworkBytesIn: int64N(instance.Network.BytesIn),
|
||||
NetworkBytesOut: int64N(instance.Network.BytesOut),
|
||||
NetworkPacketsIn: int64N(instance.Network.PacketsIn),
|
||||
NetworkPacketsOut: int64N(instance.Network.PacketsOut),
|
||||
NetworkConnectionsIn: int32N(instance.Network.ConnectionsIn),
|
||||
NetworkConnectionsOut: int32N(instance.Network.ConnectionsOut),
|
||||
NetworkErrorsIn: int64N(instance.Network.ErrorsIn),
|
||||
NetworkErrorsOut: int64N(instance.Network.ErrorsOut),
|
||||
StartTime: timeN(instance.StartTime),
|
||||
LastSeen: timeN(instance.LastSeen),
|
||||
HealthStatus: strN(instance.Health.Status),
|
||||
HealthLastCheck: timeN(instance.Health.LastCheck),
|
||||
HealthCheckCount: sql.NullInt32{Int32: int32(instance.Health.CheckCount), Valid: true},
|
||||
HealthFailureCount: sql.NullInt32{Int32: int32(instance.Health.FailureCount), Valid: true},
|
||||
})
|
||||
}
|
||||
|
||||
func (s *PostgreSQLMetricsStorage) getInstanceMetrics(ctx context.Context, serviceID string, timestamp time.Time) ([]InstanceMetrics, error) {
|
||||
query := `
|
||||
SELECT instance_id, node_id, status, cpu, memory,
|
||||
network_bytes_in, network_bytes_out, network_packets_in, network_packets_out,
|
||||
network_connections_in, network_connections_out, network_errors_in, network_errors_out,
|
||||
start_time, last_seen, health_status, health_last_check, health_check_count, health_failure_count
|
||||
FROM instance_metrics
|
||||
WHERE service_id = $1 AND timestamp = $2
|
||||
`
|
||||
|
||||
rows, err := s.db.QueryContext(ctx, query, serviceID, timestamp)
|
||||
rows, err := s.q.ListInstanceMetrics(ctx, sqlcdb.ListInstanceMetricsParams{
|
||||
ServiceID: serviceID,
|
||||
Timestamp: timestamp,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var instances []InstanceMetrics
|
||||
for rows.Next() {
|
||||
var i InstanceMetrics
|
||||
err := rows.Scan(
|
||||
&i.InstanceID, &i.NodeID, &i.Status, &i.CPU, &i.Memory,
|
||||
&i.Network.BytesIn, &i.Network.BytesOut, &i.Network.PacketsIn, &i.Network.PacketsOut,
|
||||
&i.Network.ConnectionsIn, &i.Network.ConnectionsOut, &i.Network.ErrorsIn, &i.Network.ErrorsOut,
|
||||
&i.StartTime, &i.LastSeen, &i.Health.Status, &i.Health.LastCheck, &i.Health.CheckCount, &i.Health.FailureCount,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
instances = append(instances, i)
|
||||
for _, r := range rows {
|
||||
instances = append(instances, InstanceMetrics{
|
||||
InstanceID: r.InstanceID,
|
||||
NodeID: r.NodeID.String,
|
||||
Status: r.Status.String,
|
||||
CPU: f64v(r.Cpu),
|
||||
Memory: i64v(r.Memory),
|
||||
Network: NetworkMetrics{
|
||||
BytesIn: i64v(r.NetworkBytesIn),
|
||||
BytesOut: i64v(r.NetworkBytesOut),
|
||||
PacketsIn: i64v(r.NetworkPacketsIn),
|
||||
PacketsOut: i64v(r.NetworkPacketsOut),
|
||||
ConnectionsIn: i32v(r.NetworkConnectionsIn),
|
||||
ConnectionsOut: i32v(r.NetworkConnectionsOut),
|
||||
ErrorsIn: i64v(r.NetworkErrorsIn),
|
||||
ErrorsOut: i64v(r.NetworkErrorsOut),
|
||||
},
|
||||
StartTime: r.StartTime.Time,
|
||||
LastSeen: r.LastSeen.Time,
|
||||
Health: HealthMetrics{
|
||||
Status: r.HealthStatus.String,
|
||||
LastCheck: r.HealthLastCheck.Time,
|
||||
CheckCount: int(r.HealthCheckCount.Int32),
|
||||
FailureCount: int(r.HealthFailureCount.Int32),
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
return instances, nil
|
||||
}
|
||||
|
||||
@@ -544,7 +469,9 @@ func (s *InMemoryMetricsStorage) GetServiceMetrics(ctx context.Context, serviceI
|
||||
}
|
||||
|
||||
func (s *InMemoryMetricsStorage) GetAggregatedMetrics(ctx context.Context, query MetricsQuery) (*AggregatedMetrics, error) {
|
||||
// Simplified implementation
|
||||
s.mu.RLock()
|
||||
defer s.mu.RUnlock()
|
||||
|
||||
return &AggregatedMetrics{
|
||||
Query: query,
|
||||
TimeSeries: []TimeSeriesPoint{},
|
||||
|
||||
@@ -0,0 +1,130 @@
|
||||
package metrics
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Live round-trip against a real Postgres — set METRICS_TEST_DSN (or
|
||||
// DATABASE_URL) to enable. Skipped by default so unit runs stay hermetic.
|
||||
func testDSN(t *testing.T) *sql.DB {
|
||||
t.Helper()
|
||||
dsn := os.Getenv("METRICS_TEST_DSN")
|
||||
if dsn == "" {
|
||||
dsn = os.Getenv("DATABASE_URL")
|
||||
}
|
||||
if dsn == "" {
|
||||
t.Skip("METRICS_TEST_DSN not set")
|
||||
}
|
||||
db, err := sql.Open("postgres", dsn)
|
||||
if err != nil {
|
||||
t.Fatalf("open: %v", err)
|
||||
}
|
||||
if err := db.PingContext(context.Background()); err != nil {
|
||||
t.Fatalf("ping: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { db.Close() })
|
||||
return db
|
||||
}
|
||||
|
||||
func TestPostgresStorageRoundTrip(t *testing.T) {
|
||||
db := testDSN(t)
|
||||
s := NewPostgreSQLMetricsStorage(db)
|
||||
ctx := context.Background()
|
||||
|
||||
ts := time.Now().UTC().Truncate(time.Second)
|
||||
nodeID := "test-node-" + ts.Format("150405")
|
||||
svcID := "test-svc-" + ts.Format("150405")
|
||||
|
||||
// Clean slate + teardown
|
||||
db.ExecContext(ctx, `DELETE FROM instance_metrics WHERE service_id = $1`, svcID)
|
||||
db.ExecContext(ctx, `DELETE FROM service_metrics WHERE service_id = $1`, svcID)
|
||||
db.ExecContext(ctx, `DELETE FROM node_metrics WHERE node_id = $1`, nodeID)
|
||||
t.Cleanup(func() {
|
||||
db.ExecContext(ctx, `DELETE FROM instance_metrics WHERE service_id = $1`, svcID)
|
||||
db.ExecContext(ctx, `DELETE FROM service_metrics WHERE service_id = $1`, svcID)
|
||||
db.ExecContext(ctx, `DELETE FROM node_metrics WHERE node_id = $1`, nodeID)
|
||||
})
|
||||
|
||||
node := &NodeMetrics{
|
||||
NodeID: nodeID,
|
||||
Timestamp: ts,
|
||||
CPU: CPUMetrics{UsagePercent: 42.5, UsageCores: 1.7, LoadAverage1: 0.5, LoadAverage5: 0.4, LoadAverage15: 0.3},
|
||||
Memory: MemoryMetrics{Total: 16 << 30, Used: 8 << 30, Available: 8 << 30, UsagePercent: 50.0},
|
||||
Storage: StorageMetrics{Total: 100 << 30, Used: 40 << 30, Available: 60 << 30, UsagePercent: 40.0},
|
||||
Network: NetworkMetrics{BytesIn: 1000, BytesOut: 2000, PacketsIn: 10, PacketsOut: 20, ConnectionsIn: 3, ConnectionsOut: 4, ErrorsIn: 0, ErrorsOut: 1},
|
||||
System: SystemMetrics{Uptime: 3600 * time.Second, Processes: 120, OS: "linux", Kernel: "6.8", Architecture: "amd64"},
|
||||
}
|
||||
if err := s.StoreNodeMetrics(ctx, node); err != nil {
|
||||
t.Fatalf("store node: %v", err)
|
||||
}
|
||||
// Upsert idempotent on the same key.
|
||||
if err := s.StoreNodeMetrics(ctx, node); err != nil {
|
||||
t.Fatalf("store node (repeat): %v", err)
|
||||
}
|
||||
|
||||
svc := &ServiceMetrics{
|
||||
ServiceID: svcID,
|
||||
ServiceName: "svc",
|
||||
ProjectID: "proj",
|
||||
Timestamp: ts,
|
||||
Requests: RequestMetrics{Total: 100, Success: 95, Errors: 5, AvgLatency: 12.5, P95Latency: 80, P99Latency: 120, Throughput: 3.3},
|
||||
Errors: ErrorMetrics{Total: 5, Rate: 0.05},
|
||||
Performance: PerformanceMetrics{ResponseTime: 12.5, Throughput: 3.3, Concurrency: 2, Saturation: 0.4, Utilization: 0.55},
|
||||
Resources: ResourceMetrics{CPUUsage: 42.5, MemoryUsage: 512 << 20, StorageUsage: 1 << 30, NetworkUsage: 3000, ResourceScore: 0.7},
|
||||
Instances: []InstanceMetrics{{
|
||||
InstanceID: "inst-1",
|
||||
NodeID: nodeID,
|
||||
Status: "running",
|
||||
CPU: 42.5,
|
||||
Memory: 512 << 20,
|
||||
Network: NetworkMetrics{BytesIn: 1000, BytesOut: 2000},
|
||||
StartTime: ts.Add(-time.Hour),
|
||||
LastSeen: ts,
|
||||
Health: HealthMetrics{Status: "healthy", LastCheck: ts, CheckCount: 10, FailureCount: 0},
|
||||
}},
|
||||
}
|
||||
if err := s.StoreServiceMetrics(ctx, svc); err != nil {
|
||||
t.Fatalf("store service: %v", err)
|
||||
}
|
||||
|
||||
from, to := ts.Add(-time.Minute), ts.Add(time.Minute)
|
||||
gotNodes, err := s.GetNodeMetrics(ctx, nodeID, from, to)
|
||||
if err != nil {
|
||||
t.Fatalf("get nodes: %v", err)
|
||||
}
|
||||
if len(gotNodes) != 1 {
|
||||
t.Fatalf("want 1 node row, got %d", len(gotNodes))
|
||||
}
|
||||
if got := gotNodes[0]; got.CPU.UsagePercent != 42.5 || got.System.Processes != 120 || got.System.Uptime != time.Hour {
|
||||
t.Errorf("node round-trip mismatch: %+v", got)
|
||||
}
|
||||
|
||||
gotSvcs, err := s.GetServiceMetrics(ctx, svcID, from, to)
|
||||
if err != nil {
|
||||
t.Fatalf("get services: %v", err)
|
||||
}
|
||||
if len(gotSvcs) != 1 {
|
||||
t.Fatalf("want 1 service row, got %d", len(gotSvcs))
|
||||
}
|
||||
if gotSvcs[0].Requests.P99Latency != 120 {
|
||||
t.Errorf("p99 mismatch: %v", gotSvcs[0].Requests.P99Latency)
|
||||
}
|
||||
if len(gotSvcs[0].Instances) != 1 || gotSvcs[0].Instances[0].Health.Status != "healthy" {
|
||||
t.Errorf("instance join mismatch: %+v", gotSvcs[0].Instances)
|
||||
}
|
||||
|
||||
agg, err := s.GetAggregatedMetrics(ctx, MetricsQuery{Type: "node", ID: nodeID, From: from, To: to, Interval: 5 * time.Minute})
|
||||
if err != nil {
|
||||
t.Fatalf("aggregate: %v", err)
|
||||
}
|
||||
if len(agg.TimeSeries) != 1 || agg.TimeSeries[0].Values["cpu_usage"] != 42.5 {
|
||||
t.Errorf("aggregate mismatch: %+v", agg.TimeSeries)
|
||||
}
|
||||
if agg.Summary["cpu_usage"].Avg != 42.5 {
|
||||
t.Errorf("summary mismatch: %+v", agg.Summary)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user