From f2aa9ceba81ab9931a1e4318059777c7c5516934 Mon Sep 17 00:00:00 2001 From: Tomas Dvorak Date: Sat, 3 Oct 2026 10:36:27 +0200 Subject: [PATCH] feat(infra): external monitoring stack (Beszel + Prometheus/Grafana) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Homelab monitoring design (docs/specs/2026-10-03-external-monitoring.md): Beszel agents own the two physical hosts, prometheus-pve-exporter covers every LXC/VM via the PVE API with no guest agents, and Grafana supplies graphs. Dash needs zero code changes — JSON-path widgets already carry an Authorization header and can walk Prometheus/Beszel JSON, embed widgets iframe Grafana panels, and monitors + notification transports provide the alerting layer for unmanaged hardware like the CM834 switch. infra/monitoring/ ships compose.yaml, prometheus.yml, pve.yml.example (gitignored real file), and Grafana datasource provisioning. docs/monitoring.md documents deployment and widget recipes. Authored By: TDvorak Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .gitignore | 1 + CHANGELOG.md | 8 ++ README.md | 1 + docs/monitoring.md | 119 +++++++++++++++++++ docs/specs/2026-10-03-external-monitoring.md | 81 +++++++++++++ infra/monitoring/compose.yaml | 65 ++++++++++ infra/monitoring/grafana/datasources.yaml | 10 ++ infra/monitoring/prometheus.yml | 31 +++++ infra/monitoring/pve.yml.example | 13 ++ 9 files changed, 329 insertions(+) create mode 100644 docs/monitoring.md create mode 100644 docs/specs/2026-10-03-external-monitoring.md create mode 100644 infra/monitoring/compose.yaml create mode 100644 infra/monitoring/grafana/datasources.yaml create mode 100644 infra/monitoring/prometheus.yml create mode 100644 infra/monitoring/pve.yml.example diff --git a/.gitignore b/.gitignore index b8deb7e..c85b3e5 100644 --- a/.gitignore +++ b/.gitignore @@ -15,6 +15,7 @@ data/* # env / secrets .env .env.* +infra/monitoring/pve.yml # misc .DS_Store diff --git a/CHANGELOG.md b/CHANGELOG.md index 99e1523..6b2d39e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,14 @@ All notable changes to Dash. Format follows ## [Unreleased] +### Added + +- External monitoring stack `infra/monitoring/` (Prometheus + Grafana + + prometheus-pve-exporter + Beszel hub) with a `docs/monitoring.md` guide: + Beszel agents cover the physical hosts, pve-exporter covers all LXCs/VMs + via the PVE API, and existing JSON-path/embed widgets plus monitors + consume the data — no app code required + ## [2.1.0] — 2026-09-23 ### Added diff --git a/README.md b/README.md index 50fc8d0..64687d2 100644 --- a/README.md +++ b/README.md @@ -117,6 +117,7 @@ in [docs/installation.md](docs/installation.md#dash-agent-system-monitoring). - [docs/installation.md](docs/installation.md) — Docker, binary, agent, reverse proxy, upgrades - [docs/configuration.md](docs/configuration.md) — env vars, notifications, auth, boards, operations +- [docs/monitoring.md](docs/monitoring.md) — external monitoring stack (Beszel + Prometheus/Grafana) and how Dash consumes it - [docs/widgets.md](docs/widgets.md) — widget development guide - `openapi.yaml` — API contract (generated TS client in `packages/api-client`) - [ROADMAP.md](ROADMAP.md) — design decisions and release history diff --git a/docs/monitoring.md b/docs/monitoring.md new file mode 100644 index 0000000..f57d3e3 --- /dev/null +++ b/docs/monitoring.md @@ -0,0 +1,119 @@ +# Monitoring your infrastructure + +Dash displays and alerts; it does not collect metrics beyond what +`dash-agent` pushes. For real infrastructure monitoring, run the reference +stack in [`infra/monitoring/`](../infra/monitoring/) next to Dash — it works +on any host. A small Debian LXC on the Proxmox box is a good home. + +## What watches what + +| Layer | Tool | Covers | +|---|---|---| +| Physical hosts | Beszel hub + agents | Proxmox host, Windows 11 box — CPU/mem/disk/net/temps | +| Cluster + apps | Prometheus + pve-exporter | Every LXC/VM via the PVE API (no guest agents), Dash's own `/api/metrics` | +| Dashboards | Grafana | Graphs over Prometheus, embedded into Dash via iframes | +| Up/down + alerts | Dash monitors | Router, hosts, services — with notification transports and incidents | + +Two agents total — Beszel deliberately owns only the physical machines; the +PVE API already knows everything about the guests. + +## 1. Deploy the stack + +```sh +cd infra/monitoring +cp pve.yml.example pve.yml # fill in the PVE API token (see below) +# edit prometheus.yml: replace PVE_HOST and DASH_HOST with real addresses +docker compose up -d +``` + +UIs: Prometheus `:9090`, Grafana `:3001`, Beszel `:8090`, pve-exporter `:9221`. + +### Proxmox API token (for pve-exporter) + +On the PVE host: + +```sh +pveum user add dashmon@pve --comment "read-only monitoring" +pveum aclmod / -user dashmon@pve -role PVEAuditor +``` + +Then **Datacenter → Permissions → API Tokens → Add** a token named `dash` +for `dashmon@pve`, uncheck *Privilege Separation*, copy the secret into +`pve.yml`. Token auth means no password in the file. + +### Beszel agents (physical hosts) + +1. Open Beszel `http://:8090`, create the admin account. +2. **Add System** → the hub generates the exact install command with the + key embedded — binary/systemd for the PVE host, Windows build for the + dev box (run it under NSSM or Task Scheduler for boot persistence). +3. Agents listen on `:45876`; the hub connects to them. On the same LAN + this needs no extra plumbing. + +## 2. Show it in Dash — no code required + +### Live values → JSON value widget + +Add item → widget type **JSON value**: + +**Beszel** — fields are version-dependent; browse the API to pick a path: + +- URL: `http://:8090/api/collections/systems/records` +- Header: `Authorization: Bearer ` — mint a superuser token/key in + the PocketBase admin at `http://:8090/_/` +- Path: e.g. `items[0].info.cpu` (verify in the JSON response) + +**Prometheus** — instant queries return JSON: + +- URL: `http://:9090/api/v1/query?query=pve_cpu_usage_ratio` +- Path: `data.result[0].value[1]` +- Useful queries: `pve_cpu_usage_ratio`, `pve_memory_usage_ratio`, + `pve_disk_usage_ratio`, `pve_up` + +### Graphs → Embed widget + +Grafana dashboard → share a panel → **Embed** gives a `d-solo` URL. Paste it +into a Dash **Embed** widget (append `&kiosk` for chrome-free panels). The +shipped compose enables anonymous Viewer + embedding, so iframes render +without login on the LAN — remove the `GF_AUTH_ANONYMOUS_*` and +`GF_SECURITY_ALLOW_EMBEDDING` env lines if you'd rather require sign-in, and +use Grafana *public dashboard* links instead. + +The whole Beszel UI also iframes cleanly if you want its charts in a tile. + +## 3. Alert on outages → Dash monitors + +The point of the exercise: when the unmanaged switch hangs, everything +*behind* it drops at once while router-side checks stay green. That +co-failure pattern — timestamped, pushed to your phone — is as close to +"the switch died" as an unmanaged box allows. + +Recommended monitors: + +| Target | Type | Why | +|---|---|---| +| Router / gateway IP | Ping | Upstream reference point | +| Proxmox host | Ping + HTTP `:8006` | Behind the switch | +| Each LXC's service | HTTP/TCP | Per-app liveness | +| Win11 box | TCP `:3389` (RDP) | Behind the switch | +| Prometheus/Grafana/Beszel | HTTP | Watch the watchers | + +Settings → Notifications: configure ntfy/Telegram/etc. once and every +down-transition alerts you with an incident record. + +**Possible switch fix, unproven:** cheap-switch hangs are often EEE +(energy-efficient ethernet) negotiation bugs. `ethtool --set-eee eee +off` on the host NICs is a reasonable mitigation. Failing that, the usual +suspects are heat and the power brick. + +## 4. Files + +``` +infra/monitoring/compose.yaml stack definition +infra/monitoring/prometheus.yml scrape config (edit hostnames) +infra/monitoring/pve.yml.example PVE token template → copy to pve.yml +infra/monitoring/grafana/datasources.yaml Prometheus source, auto-provisioned +``` + +`pve.yml` holds a credential and is gitignored. Stack design rationale: +[`docs/specs/2026-10-03-external-monitoring.md`](specs/2026-10-03-external-monitoring.md). diff --git a/docs/specs/2026-10-03-external-monitoring.md b/docs/specs/2026-10-03-external-monitoring.md new file mode 100644 index 0000000..5e1a3d2 --- /dev/null +++ b/docs/specs/2026-10-03-external-monitoring.md @@ -0,0 +1,81 @@ +# External Monitoring Stack — Design + +Date: 2026-10-03 +Status: approved in chat 2026-10-03 — implementing + +## 1. Goal + +Monitor a homelab (one Proxmox host with LXCs, one always-on Windows 11 dev +box, an unmanaged Ugreen CM834 switch, a router) with the collection layer +living **outside** Dash. Dash consumes data for display and owns up/down +alerting. Trigger: an unmanaged switch hung overnight with no way to know +when or what failed. + +## 2. Lane split — why two engines + +Two engines were requested. They earn their place only if lanes don't +overlap: + +| Tool | Lane | Scope | +|---|---|---| +| Beszel (hub + agents) | The two **physical** machines: PVE host, Win11 | Agent push, Windows support, instant host tiles, standalone alerts | +| Prometheus + `prometheus-pve-exporter` | Everything **scrapable**: all LXCs/VMs via the PVE API (no guest agents), Dash's own `/api/metrics` | System of record, PromQL | +| Grafana | Graphing front for Prometheus | Panel iframes for Dash | + +Deliberately absent: `node_exporter`, `windows_exporter` — Beszel already +owns the physical hosts. Two agents total, not two per machine. + +## 3. Dash integration — zero code changes + +Existing primitives cover the whole display layer: + +- **JSON-path widget** → Beszel PocketBase API and Prometheus + `/api/v1/query` (`data.result[0].value[1]` shape already walkable; + `header` config field carries `Authorization: Bearer …`) +- **Embed widget** → Grafana panel URLs (anonymous Viewer + embedding + enabled on the LAN) and the Beszel UI +- **Monitors** → ping/TCP/HTTP checks + notification transports + + auto-incidents = the "something died at 03:12" alarm the user lacked +- **Service links** → per-console deep links + +## 4. The switch problem + +The Ugreen CM834 is unmanaged: no IP, no SNMP, no telemetry. Root cause is +unobservable. What monitoring delivers instead: + +- Monitors on router IP, PVE host, and one service per LXC — all reached + *through* the switch. When it hangs, behind-switch targets fail together + while router-side checks stay green: a timestamped "it was the switch" + signature plus an alert to ntfy/Telegram. +- Plausible mitigation (not proven): disable EEE on host NICs — + `ethtool --set-eee eee off`. Cheap-switch EEE hangs are a known + failure mode. + +## 5. Deliverables + +``` +infra/monitoring/ + compose.yaml # prometheus + grafana + pve-exporter + beszel-hub + prometheus.yml # scrape config: pve (via exporter) + dash /api/metrics + pve.yml.example # PVE API token config (copy → pve.yml, gitignored) + grafana/datasources.yaml # auto-provisions the Prometheus source +docs/monitoring.md # deploy guide + Dash widget/monitor recipes +``` + +No application code. No schema changes. Stack runs anywhere — an LXC on the +Proxmox host is the suggested home. + +## 6. Security notes + +- Grafana anonymous **Viewer** + `allow_embedding` are set for LAN iframe + use; both are called out in the guide and removable. +- `pve.yml` carries an API token — `pve.yml.example` is committed, the real + file is documented as gitignored-adjacent (lives next to the compose, + outside the repo if preferred). +- Widget secrets already live in item config JSON per the original spec. + +## 7. Verification + +- `docker compose -f infra/monitoring/compose.yaml config` valid +- `promtool check config prometheus.yml` (dockerized promtool) valid +- Docs-only otherwise; repo gate (typecheck/lint/build/go tests) unchanged diff --git a/infra/monitoring/compose.yaml b/infra/monitoring/compose.yaml new file mode 100644 index 0000000..814b6a9 --- /dev/null +++ b/infra/monitoring/compose.yaml @@ -0,0 +1,65 @@ +# External monitoring stack — independent of the Dash container. +# Run on any host (a small Debian LXC on Proxmox works well): +# cp pve.yml.example pve.yml # fill in your PVE API token +# docker compose up -d +# All state lives in named volumes. +name: dash-monitoring + +services: + prometheus: + image: prom/prometheus:latest + container_name: prometheus + restart: unless-stopped + command: + - --config.file=/etc/prometheus/prometheus.yml + - --storage.tsdb.retention.time=30d + - --web.enable-lifecycle # POST /-/reload applies config changes + volumes: + - ./prometheus.yml:/etc/prometheus/prometheus.yml:ro + - prometheus-data:/prometheus + ports: + - "9090:9090" + + grafana: + image: grafana/grafana:latest + container_name: grafana + restart: unless-stopped + environment: + # LAN convenience for Dash iframe widgets: read-only anonymous access. + # Anyone on your network can VIEW dashboards without logging in. + # Delete these three lines to require sign-in instead. + GF_AUTH_ANONYMOUS_ENABLED: "true" + GF_AUTH_ANONYMOUS_ORG_ROLE: Viewer + GF_SECURITY_ALLOW_EMBEDDING: "true" # required for iframe embeds + GF_USERS_ALLOW_SIGN_UP: "false" + volumes: + - ./grafana/datasources.yaml:/etc/grafana/provisioning/datasources/datasources.yaml:ro + - grafana-data:/var/lib/grafana + ports: + - "3001:3000" + depends_on: + - prometheus + + pve-exporter: + image: prompve/prometheus-pve-exporter:latest + container_name: pve-exporter + restart: unless-stopped + command: ["--config.file", "/etc/pve.yml"] + volumes: + - ./pve.yml:/etc/pve.yml:ro + ports: + - "9221:9221" + + beszel-hub: + image: henrygd/beszel:latest + container_name: beszel + restart: unless-stopped + volumes: + - beszel-data:/beszel_data + ports: + - "8090:8090" + +volumes: + prometheus-data: + grafana-data: + beszel-data: diff --git a/infra/monitoring/grafana/datasources.yaml b/infra/monitoring/grafana/datasources.yaml new file mode 100644 index 0000000..03085ae --- /dev/null +++ b/infra/monitoring/grafana/datasources.yaml @@ -0,0 +1,10 @@ +# Auto-provisions the Prometheus datasource so Grafana works on first boot. +apiVersion: 1 + +datasources: + - name: Prometheus + type: prometheus + access: proxy + url: http://prometheus:9090 + isDefault: true + editable: true diff --git a/infra/monitoring/prometheus.yml b/infra/monitoring/prometheus.yml new file mode 100644 index 0000000..a2588b7 --- /dev/null +++ b/infra/monitoring/prometheus.yml @@ -0,0 +1,31 @@ +# Prometheus scrape config for the Dash-adjacent monitoring stack. +# Adjust the host addresses to your LAN before starting. + +global: + scrape_interval: 30s + evaluation_interval: 30s + +scrape_configs: + # Proxmox cluster: node + every LXC/VM via the PVE API. + # No agents inside guests — pve-exporter asks the API on :8006. + - job_name: pve + metrics_path: /pve + params: + module: [default] + cluster: ["1"] + node: ["1"] + static_configs: + - targets: ["PVE_HOST:8006"] # <- your Proxmox host + relabel_configs: + - source_labels: [__address__] + target_label: __param_target + - source_labels: [__param_target] + target_label: instance + - target_label: __address__ + replacement: pve-exporter:9221 + + # Dash itself — it exposes Prometheus-format metrics already. + - job_name: dash + metrics_path: /api/metrics + static_configs: + - targets: ["DASH_HOST:3000"] # <- wherever Dash runs (CasaOS box, etc.) diff --git a/infra/monitoring/pve.yml.example b/infra/monitoring/pve.yml.example new file mode 100644 index 0000000..9decbc7 --- /dev/null +++ b/infra/monitoring/pve.yml.example @@ -0,0 +1,13 @@ +# prometheus-pve-exporter config — copy to pve.yml and fill in. +# pve.yml contains a credential: keep it out of git. +# +# Recommended setup on the Proxmox host: +# pveum user add dashmon@pve --comment "read-only monitoring" +# pveum aclmod / -user dashmon@pve -role PVEAuditor +# Datacenter -> Permissions -> API Tokens -> add token "dash" for dashmon@pve, +# uncheck "Privilege Separation" (the token inherits the user's PVEAuditor role) +default: + user: dashmon@pve + token_name: dash + token_value: "xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" + verify_ssl: false # PVE ships self-signed certs on LAN