mirror of
https://github.com/Dvorinka/Dash.git
synced 2026-10-28 15:51:27 +00:00
feat(api): phase 5 — alerts, agent depth, subdomains, auth, boards
P5b: notifier transport registry (webhook/slack/discord/telegram/ gotify/ntfy/smtp via net/smtp), per-monitor+domain alert rules (failsBeforeDown, latencyWarn, certDays, mute). P5c: per-container cpu/mem, optional SMART/ZFS/GPU exec-or-skip collectors, crt.sh subdomain discovery with daily sweep. P5d: opt-in auth (bcrypt, sqlite sessions, 7d rotating cookie, login rate limit, public-route bypass) and multiple boards (boards table, sections.board_id, /api/boards CRUD, slug routing). Authored By: TDvorak <[email protected]> Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
1 parent
8973be3b41
commit
f3143af351
33 files changed
+2772
-126
No files matched your search
@@ -0,0 +1,121 @@
|
||||
package main
|
||||
|
||||
// Docker collector: container liveness plus per-container CPU/memory from
|
||||
// /containers/{id}/stats?stream=false. All best-effort — no socket, no data.
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
const dockerSock = "/var/run/docker.sock"
|
||||
|
||||
var dockerClient = &http.Client{
|
||||
Timeout: 4 * time.Second,
|
||||
Transport: &http.Transport{
|
||||
DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) {
|
||||
return (&net.Dialer{Timeout: 2 * time.Second}).DialContext(ctx, "unix", dockerSock)
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
// dockerStatsJSON is the subset of /containers/{id}/stats we consume.
|
||||
type dockerStatsJSON struct {
|
||||
CPUStats struct {
|
||||
CPUUsage struct {
|
||||
TotalUsage uint64 `json:"total_usage"`
|
||||
} `json:"cpu_usage"`
|
||||
SystemCPUUsage uint64 `json:"system_cpu_usage"`
|
||||
OnlineCPUs uint32 `json:"online_cpus"`
|
||||
} `json:"cpu_stats"`
|
||||
PreCPUStats struct {
|
||||
CPUUsage struct {
|
||||
TotalUsage uint64 `json:"total_usage"`
|
||||
} `json:"cpu_usage"`
|
||||
SystemCPUUsage uint64 `json:"system_cpu_usage"`
|
||||
} `json:"precpu_stats"`
|
||||
MemoryStats struct {
|
||||
Usage uint64 `json:"usage"`
|
||||
Limit uint64 `json:"limit"`
|
||||
} `json:"memory_stats"`
|
||||
}
|
||||
|
||||
// containerCPUPct applies the Docker dashboard formula: cpu delta over
|
||||
// system delta, scaled by online CPUs. 0 when deltas don't advance.
|
||||
func containerCPUPct(s *dockerStatsJSON) float64 {
|
||||
cpuDelta := float64(s.CPUStats.CPUUsage.TotalUsage) - float64(s.PreCPUStats.CPUUsage.TotalUsage)
|
||||
sysDelta := float64(s.CPUStats.SystemCPUUsage) - float64(s.PreCPUStats.SystemCPUUsage)
|
||||
online := s.CPUStats.OnlineCPUs
|
||||
if online == 0 {
|
||||
online = 1
|
||||
}
|
||||
if cpuDelta <= 0 || sysDelta <= 0 {
|
||||
return 0
|
||||
}
|
||||
return cpuDelta / sysDelta * float64(online) * 100
|
||||
}
|
||||
|
||||
// readContainers lists containers and fans out to per-container stats.
|
||||
// Stats fetches are bounded: 8 concurrent, each under the client timeout.
|
||||
func readContainers() []containerStat {
|
||||
if _, err := net.DialTimeout("unix", dockerSock, time.Second); err != nil {
|
||||
return nil
|
||||
}
|
||||
res, err := dockerClient.Get("http://d/containers/json?all=1")
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
defer res.Body.Close()
|
||||
var raw []struct {
|
||||
ID string `json:"Id"`
|
||||
Names []string `json:"Names"`
|
||||
State string `json:"State"`
|
||||
}
|
||||
if json.NewDecoder(io.LimitReader(res.Body, 4<<20)).Decode(&raw) != nil {
|
||||
return nil
|
||||
}
|
||||
out := make([]containerStat, len(raw))
|
||||
var wg sync.WaitGroup
|
||||
sem := make(chan struct{}, 8)
|
||||
for i, r := range raw {
|
||||
name := ""
|
||||
if len(r.Names) > 0 {
|
||||
name = strings.TrimPrefix(r.Names[0], "/")
|
||||
}
|
||||
out[i] = containerStat{Name: name, State: r.State}
|
||||
if r.State != "running" || r.ID == "" {
|
||||
continue
|
||||
}
|
||||
wg.Add(1)
|
||||
sem <- struct{}{}
|
||||
go func(i int, id string) {
|
||||
defer wg.Done()
|
||||
defer func() { <-sem }()
|
||||
fillContainerStats(&out[i], id)
|
||||
}(i, r.ID)
|
||||
}
|
||||
wg.Wait()
|
||||
return out
|
||||
}
|
||||
|
||||
// fillContainerStats fetches one container's one-shot stats into cs.
|
||||
func fillContainerStats(cs *containerStat, id string) {
|
||||
res, err := dockerClient.Get("http://d/containers/" + id + "/stats?stream=false")
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
defer res.Body.Close()
|
||||
var st dockerStatsJSON
|
||||
if json.NewDecoder(io.LimitReader(res.Body, 4<<20)).Decode(&st) != nil {
|
||||
return
|
||||
}
|
||||
cs.CPU = containerCPUPct(&st)
|
||||
cs.MemUsed = float64(st.MemoryStats.Usage)
|
||||
cs.MemLimit = float64(st.MemoryStats.Limit)
|
||||
}
|
||||
@@ -0,0 +1,230 @@
|
||||
package main
|
||||
|
||||
// Optional collectors: SMART (smartctl), ZFS (zpool), GPU (nvidia-smi or
|
||||
// /sys/class/drm). Every collector is exec-or-skip — missing tools or
|
||||
// permissions produce nil, never errors, and each exec carries a timeout so a
|
||||
// wedged tool can't stall reporting.
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// runOut executes a command with a timeout; empty output on any failure.
|
||||
func runOut(timeout time.Duration, name string, args ...string) []byte {
|
||||
if _, err := exec.LookPath(name); err != nil {
|
||||
return nil
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), timeout)
|
||||
defer cancel()
|
||||
out, err := exec.CommandContext(ctx, name, args...).Output()
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// ---------- SMART ----------
|
||||
|
||||
type smartctlScan struct {
|
||||
Devices []struct {
|
||||
Name string `json:"name"`
|
||||
} `json:"devices"`
|
||||
}
|
||||
|
||||
type smartctlAll struct {
|
||||
ModelName string `json:"model_name"`
|
||||
SmartStatus *struct {
|
||||
Passed bool `json:"passed"`
|
||||
} `json:"smart_status"`
|
||||
Temperature *struct {
|
||||
Current float64 `json:"current"`
|
||||
} `json:"temperature"`
|
||||
}
|
||||
|
||||
// readSMART scans devices via smartctl -j --scan, then reads each drive.
|
||||
// Needs smartmontools installed and usually root or sudo rights for /dev.
|
||||
func readSMART() []smartDisk {
|
||||
scan := runOut(5*time.Second, "smartctl", "-j", "--scan")
|
||||
if scan == nil {
|
||||
return nil
|
||||
}
|
||||
var sc smartctlScan
|
||||
if json.Unmarshal(scan, &sc) != nil || len(sc.Devices) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := make([]smartDisk, 0, len(sc.Devices))
|
||||
for i, d := range sc.Devices {
|
||||
if i >= 8 { // bound the fan-out
|
||||
break
|
||||
}
|
||||
b := runOut(4*time.Second, "smartctl", "-j", "-a", d.Name)
|
||||
if b == nil {
|
||||
continue
|
||||
}
|
||||
var a smartctlAll
|
||||
if json.Unmarshal(b, &a) != nil {
|
||||
continue
|
||||
}
|
||||
sd := smartDisk{Device: d.Name, Model: a.ModelName}
|
||||
if a.SmartStatus != nil {
|
||||
p := a.SmartStatus.Passed
|
||||
sd.Passed = &p
|
||||
}
|
||||
if a.Temperature != nil {
|
||||
sd.TempC = a.Temperature.Current
|
||||
}
|
||||
out = append(out, sd)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// ---------- ZFS ----------
|
||||
|
||||
// readZFS prefers `zpool list -j` (OpenZFS 2.3+), falls back to the stable
|
||||
// `-H -p` text format. Parse failures degrade to nil.
|
||||
func readZFS() []zfsPool {
|
||||
if b := runOut(4*time.Second, "zpool", "list", "-j", "--json-int",
|
||||
"-o", "name,health,size,free"); b != nil {
|
||||
if pools := parseZpoolJSON(b); pools != nil {
|
||||
return pools
|
||||
}
|
||||
}
|
||||
return parseZpoolText(runOut(4*time.Second,
|
||||
"zpool", "list", "-H", "-p", "-o", "name,health,size,free"))
|
||||
}
|
||||
|
||||
// parseZpoolJSON reads {"pools":{"name":{"properties":{...}}}}.
|
||||
func parseZpoolJSON(b []byte) []zfsPool {
|
||||
var doc struct {
|
||||
Pools map[string]struct {
|
||||
Properties map[string]struct {
|
||||
Value any `json:"value"`
|
||||
} `json:"properties"`
|
||||
} `json:"pools"`
|
||||
}
|
||||
if json.Unmarshal(b, &doc) != nil || len(doc.Pools) == 0 {
|
||||
return nil
|
||||
}
|
||||
var out []zfsPool
|
||||
num := func(v any) float64 {
|
||||
switch n := v.(type) {
|
||||
case float64:
|
||||
return n
|
||||
case string:
|
||||
f, _ := strconv.ParseFloat(n, 64)
|
||||
return f
|
||||
}
|
||||
return 0
|
||||
}
|
||||
for name, p := range doc.Pools {
|
||||
z := zfsPool{Name: name}
|
||||
if v, ok := p.Properties["health"]; ok {
|
||||
if s, ok := v.Value.(string); ok {
|
||||
z.Health = s
|
||||
}
|
||||
}
|
||||
if v, ok := p.Properties["size"]; ok {
|
||||
z.Size = num(v.Value)
|
||||
}
|
||||
if v, ok := p.Properties["free"]; ok {
|
||||
z.Free = num(v.Value)
|
||||
}
|
||||
out = append(out, z)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// parseZpoolText reads `zpool list -H -p` rows: name\thealth\tsize\tfree.
|
||||
func parseZpoolText(b []byte) []zfsPool {
|
||||
var out []zfsPool
|
||||
for _, line := range strings.Split(string(b), "\n") {
|
||||
f := strings.Split(line, "\t")
|
||||
if len(f) < 4 {
|
||||
continue
|
||||
}
|
||||
size, _ := strconv.ParseFloat(f[2], 64)
|
||||
free, _ := strconv.ParseFloat(f[3], 64)
|
||||
out = append(out, zfsPool{Name: f[0], Health: f[1], Size: size, Free: free})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// ---------- GPU ----------
|
||||
|
||||
// readGPU prefers nvidia-smi (CSV query); otherwise walks /sys/class/drm for
|
||||
// AMD/Intel cards exposing gpu_busy_percent and hwmon temps.
|
||||
func readGPU() []gpuStat {
|
||||
if _, err := exec.LookPath("nvidia-smi"); err == nil {
|
||||
return readNvidiaGPU()
|
||||
}
|
||||
return readGPUSysfs()
|
||||
}
|
||||
|
||||
func readNvidiaGPU() []gpuStat {
|
||||
return parseNvidiaCSV(runOut(4*time.Second, "nvidia-smi",
|
||||
"--query-gpu=name,temperature.gpu,utilization.gpu,memory.used,memory.total",
|
||||
"--format=csv,noheader,nounits"))
|
||||
}
|
||||
|
||||
// parseNvidiaCSV reads rows: name, tempC, util%, memUsed MiB, memTotal MiB.
|
||||
func parseNvidiaCSV(b []byte) []gpuStat {
|
||||
var out []gpuStat
|
||||
for _, line := range strings.Split(strings.TrimSpace(string(b)), "\n") {
|
||||
f := strings.Split(line, ",")
|
||||
if len(f) < 5 {
|
||||
continue
|
||||
}
|
||||
g := gpuStat{Name: strings.TrimSpace(f[0])}
|
||||
g.TempC, _ = strconv.ParseFloat(strings.TrimSpace(f[1]), 64)
|
||||
g.UtilPct, _ = strconv.ParseFloat(strings.TrimSpace(f[2]), 64)
|
||||
g.MemUsed, _ = strconv.ParseFloat(strings.TrimSpace(f[3]), 64)
|
||||
g.MemTotal, _ = strconv.ParseFloat(strings.TrimSpace(f[4]), 64)
|
||||
g.MemUsed *= 1 << 20
|
||||
g.MemTotal *= 1 << 20
|
||||
out = append(out, g)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// readGPUSysfs pulls util/temp for each card from DRM sysfs.
|
||||
func readGPUSysfs() []gpuStat {
|
||||
cards, _ := filepath.Glob("/sys/class/drm/card[0-9]")
|
||||
var out []gpuStat
|
||||
for _, card := range cards {
|
||||
dev := card + "/device"
|
||||
g := gpuStat{Name: filepath.Base(card)}
|
||||
if b, err := os.ReadFile(dev + "/gpu_busy_percent"); err == nil {
|
||||
g.UtilPct, _ = strconv.ParseFloat(strings.TrimSpace(string(b)), 64)
|
||||
}
|
||||
// hwmon: first temp1_input under device/hwmon/hwmonN.
|
||||
if hws, err := filepath.Glob(dev + "/hwmon/hwmon*"); err == nil {
|
||||
for _, hw := range hws {
|
||||
if b, err := os.ReadFile(hw + "/temp1_input"); err == nil {
|
||||
if milli, err := strconv.ParseFloat(strings.TrimSpace(string(b)), 64); err == nil {
|
||||
g.TempC = milli / 1000
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
// Driver name via uevent DRIVER= for a friendlier label.
|
||||
if b, err := os.ReadFile(dev + "/uevent"); err == nil {
|
||||
for _, line := range strings.Split(string(b), "\n") {
|
||||
if v, ok := strings.CutPrefix(line, "DRIVER="); ok {
|
||||
g.Name += " (" + v + ")"
|
||||
}
|
||||
}
|
||||
}
|
||||
if g.UtilPct > 0 || g.TempC > 0 {
|
||||
out = append(out, g)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -9,13 +9,11 @@ package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"net"
|
||||
"net/http"
|
||||
"os"
|
||||
"runtime"
|
||||
@@ -96,11 +94,40 @@ type sample struct {
|
||||
Load15 float64 `json:"load15"`
|
||||
Temps map[string]float64 `json:"temps,omitempty"`
|
||||
Containers []containerStat `json:"containers,omitempty"`
|
||||
Smart []smartDisk `json:"smart,omitempty"`
|
||||
ZFS []zfsPool `json:"zfs,omitempty"`
|
||||
GPU []gpuStat `json:"gpu,omitempty"`
|
||||
}
|
||||
|
||||
type containerStat struct {
|
||||
Name string `json:"name"`
|
||||
State string `json:"state"`
|
||||
Name string `json:"name"`
|
||||
State string `json:"state"`
|
||||
CPU float64 `json:"cpu,omitempty"`
|
||||
MemUsed float64 `json:"memUsed,omitempty"`
|
||||
MemLimit float64 `json:"memLimit,omitempty"`
|
||||
}
|
||||
|
||||
// Optional-collector shapes — mirror api.SmartDisk/ZFSPool/GPUStat.
|
||||
type smartDisk struct {
|
||||
Device string `json:"device"`
|
||||
Model string `json:"model"`
|
||||
Passed *bool `json:"passed,omitempty"`
|
||||
TempC float64 `json:"tempC,omitempty"`
|
||||
}
|
||||
|
||||
type zfsPool struct {
|
||||
Name string `json:"name"`
|
||||
Health string `json:"health"`
|
||||
Size float64 `json:"size"`
|
||||
Free float64 `json:"free"`
|
||||
}
|
||||
|
||||
type gpuStat struct {
|
||||
Name string `json:"name"`
|
||||
TempC float64 `json:"tempC,omitempty"`
|
||||
UtilPct float64 `json:"utilPct,omitempty"`
|
||||
MemUsed float64 `json:"memUsed,omitempty"`
|
||||
MemTotal float64 `json:"memTotal,omitempty"`
|
||||
}
|
||||
|
||||
type collector struct {
|
||||
@@ -141,6 +168,9 @@ func (c *collector) sample(interval int) sample {
|
||||
s.UptimeS = readUptime()
|
||||
s.Temps = readTemps()
|
||||
s.Containers = readContainers()
|
||||
s.Smart = readSMART()
|
||||
s.ZFS = readZFS()
|
||||
s.GPU = readGPU()
|
||||
return s
|
||||
}
|
||||
|
||||
@@ -325,38 +355,4 @@ func readTemps() map[string]float64 {
|
||||
return out
|
||||
}
|
||||
|
||||
// readContainers lists Docker containers via the unix socket when present.
|
||||
func readContainers() []containerStat {
|
||||
if _, err := os.Stat("/var/run/docker.sock"); err != nil {
|
||||
return nil
|
||||
}
|
||||
client := &http.Client{
|
||||
Timeout: 3 * time.Second,
|
||||
Transport: &http.Transport{
|
||||
DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) {
|
||||
return (&net.Dialer{}).DialContext(ctx, "unix", "/var/run/docker.sock")
|
||||
},
|
||||
},
|
||||
}
|
||||
res, err := client.Get("http://d/containers/json?all=1")
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
defer res.Body.Close()
|
||||
var raw []struct {
|
||||
Names []string `json:"Names"`
|
||||
State string `json:"State"`
|
||||
}
|
||||
if json.NewDecoder(res.Body).Decode(&raw) != nil {
|
||||
return nil
|
||||
}
|
||||
out := make([]containerStat, 0, len(raw))
|
||||
for _, r := range raw {
|
||||
name := ""
|
||||
if len(r.Names) > 0 {
|
||||
name = strings.TrimPrefix(r.Names[0], "/")
|
||||
}
|
||||
out = append(out, containerStat{Name: name, State: r.State})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
@@ -0,0 +1,48 @@
|
||||
package main
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestContainerCPUPct(t *testing.T) {
|
||||
var s dockerStatsJSON
|
||||
s.CPUStats.CPUUsage.TotalUsage = 300
|
||||
s.PreCPUStats.CPUUsage.TotalUsage = 100
|
||||
s.CPUStats.SystemCPUUsage = 1000
|
||||
s.PreCPUStats.SystemCPUUsage = 500
|
||||
s.CPUStats.OnlineCPUs = 4
|
||||
if got := containerCPUPct(&s); got != 160 {
|
||||
t.Fatalf("expected 160%%, got %v", got)
|
||||
}
|
||||
// No deltas -> 0, no div-by-zero.
|
||||
s.PreCPUStats.CPUUsage.TotalUsage = s.CPUStats.CPUUsage.TotalUsage
|
||||
s.PreCPUStats.SystemCPUUsage = s.CPUStats.SystemCPUUsage
|
||||
if got := containerCPUPct(&s); got != 0 {
|
||||
t.Fatalf("expected 0, got %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseZpoolText(t *testing.T) {
|
||||
out := parseZpoolText([]byte("tank\tONLINE\t1000000000\t400000000\nfast\tDEGRADED\t500\t100\n"))
|
||||
if len(out) != 2 || out[0].Name != "tank" || out[0].Size != 1e9 || out[1].Health != "DEGRADED" {
|
||||
t.Fatalf("bad parse: %+v", out)
|
||||
}
|
||||
if parseZpoolText(nil) != nil {
|
||||
t.Fatal("nil input should yield nil")
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseZpoolJSON(t *testing.T) {
|
||||
b := []byte(`{"output_version":{"major":0},"pools":{"tank":{"name":"tank","properties":{
|
||||
"health":{"value":"ONLINE"},"size":{"value":1000000000},"free":{"value":250000000}}}}}`)
|
||||
out := parseZpoolJSON(b)
|
||||
if len(out) != 1 || out[0].Health != "ONLINE" || out[0].Free != 250000000 {
|
||||
t.Fatalf("bad parse: %+v", out)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseNvidiaCSV(t *testing.T) {
|
||||
out := parseNvidiaCSV([]byte("RTX 3080, 62, 14, 2048, 10240\n"))
|
||||
if len(out) != 1 || out[0].Name != "RTX 3080" || out[0].TempC != 62 ||
|
||||
out[0].MemUsed != 2048*1<<20 || out[0].MemTotal != 10240*1<<20 {
|
||||
t.Fatalf("bad parse: %+v", out)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user