diff --git a/SECURITY.md b/SECURITY.md index c51147c1..6890dfe2 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -33,18 +33,18 @@ small, reversible controls and narrow exceptions; benchmark before adopting expe ## Architecture and trust boundaries -| Boundary | Current design | Limitation | -| --------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| Administration | Key-based SSH; root key login retained for Colmena/MicroVM deployment; only keys declared in Nix are honoured, and no password or keyboard-interactive login; remote-builder host keys only reach `nix-daemon` | Deployment credentials are root-equivalent file keys, usable by any same-user process through the desktop SSH agent; one stolen key reaches every host | -| Host network | Shared nftables firewall trusts the home LAN, tailnet interface and local Podman bridge; IPv6 link-local is accepted | These are broad trust zones, not service-level least privilege; router exposure and tailnet ACLs require separate verification | -| Monitoring | node/smartctl exporters bind the host LAN IPv4 (9100/9633); firewall rules restrict exporter ports 9100, 9835 and 9633 to the monitoring host over IPv4 | nvidia_gpu_exporter (9835) still binds wildcard; the IPv6 denial for those three ports is in the shared firewall; current running hosts require separate verification | -| VM hosts | shoryu, shushou and youko run guests behind `br0`; ruby/kana are libvirt guests | A bridge is not network isolation; host compromise affects its guests | -| k3s | Three control-plane and three worker MicroVMs; Cilium, API VIP and NFS CSI | Cluster-admin, privileged/host-mounted pods and writable NFS data are powerful trust boundaries | -| MicroVM storage | Host store shared read-only through virtiofs; guest `/etc`, `/var` and `/home` persisted in separate images | Read-only store does not isolate the host-side virtiofs implementation or protect writable guest state | -| Desktop apps | Selected apps use nixpak/bubblewrap; Podman is rootless | Rules and shared directories vary per app; user namespaces are needed for these sandboxes | -| Secrets | agenix declarations here, encrypted data in the private nix-secrets repository | Recipients and plaintext file permissions must be reviewed; root on a recipient can read its decrypted secrets | -| Supply chain | Locked flake inputs; explicit binary-cache trust keys on NixOS; normal user is not a Nix trusted-user | A trusted cache or imported Nix module can supply code executed as root; a lock file is not a security audit | -| Recovery | NixOS generations, btrbk/restic, hourly read-only snapshots of the REST store, and documented backup procedures | A rollback does not undo compromised data or secrets; backup configuration alone does not prove restore works; the offsite immutable copy is still planned | +| Boundary | Current design | Limitation | +| --------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| Administration | Key-based SSH; root key login retained for Colmena/MicroVM deployment; only keys declared in Nix are honoured, and no password or keyboard-interactive login; remote-builder host keys only reach `nix-daemon` | Deployment credentials are root-equivalent file keys, usable by any same-user process through the desktop SSH agent; one stolen key reaches every host | +| Host network | Shared nftables firewall trusts the home LAN, tailnet interface and local Podman bridge; IPv6 link-local is accepted | These are broad trust zones, not service-level least privilege; router exposure and tailnet ACLs require separate verification | +| Monitoring | node/smartctl exporters bind the host LAN IPv4 (9100/9633); firewall rules restrict exporter ports 9100, 9835 and 9633 to the monitoring host over IPv4. RAPL counters are exposed only on x86 server hosts, to `node-exporter` through the `rapl` group. | nvidia_gpu_exporter (9835) still binds wildcard; the IPv6 denial for those three ports is in the shared firewall; current running hosts require separate verification. The node_exporter account can measure package energy for local side channels; interactive users, desktops, and guests keep the kernel-default root-only permission. | +| VM hosts | shoryu, shushou and youko run guests behind `br0`; ruby/kana are libvirt guests | A bridge is not network isolation; host compromise affects its guests | +| k3s | Three control-plane and three worker MicroVMs; Cilium, API VIP and NFS CSI | Cluster-admin, privileged/host-mounted pods and writable NFS data are powerful trust boundaries | +| MicroVM storage | Host store shared read-only through virtiofs; guest `/etc`, `/var` and `/home` persisted in separate images | Read-only store does not isolate the host-side virtiofs implementation or protect writable guest state | +| Desktop apps | Selected apps use nixpak/bubblewrap; Podman is rootless | Rules and shared directories vary per app; user namespaces are needed for these sandboxes | +| Secrets | agenix declarations here, encrypted data in the private nix-secrets repository | Recipients and plaintext file permissions must be reviewed; root on a recipient can read its decrypted secrets | +| Supply chain | Locked flake inputs; explicit binary-cache trust keys on NixOS; normal user is not a Nix trusted-user | A trusted cache or imported Nix module can supply code executed as root; a lock file is not a security audit | +| Recovery | NixOS generations, btrbk/restic, hourly read-only snapshots of the REST store, and documented backup procedures | A rollback does not undo compromised data or secrets; backup configuration alone does not prove restore works; the offsite immutable copy is still planned | Sources: [host layout](hosts/README.md), [k3s deployment](hosts/k8s/README.md), [firewall](modules/nixos/base/networking/firewall.nix), [SSH](modules/nixos/base/ssh.nix), diff --git a/hosts/12kingdoms-youko/homelab-services/grafana/dashboards/README.md b/hosts/12kingdoms-youko/homelab-services/grafana/dashboards/README.md index 842f3864..b354edb0 100644 --- a/hosts/12kingdoms-youko/homelab-services/grafana/dashboards/README.md +++ b/hosts/12kingdoms-youko/homelab-services/grafana/dashboards/README.md @@ -8,6 +8,11 @@ on the `smartctl_exporter` metrics. 4. `gpu.json` — GPU utilization, VRAM, and temperature. 5. `nut-exporter.json` — UPS status (battery, load, runtime) from the NUT exporter. +6. `power.json` — Aquamarine's CPU package/core/uncore power from the node_exporter RAPL collector, + plus GPU power from hwmon. This is silicon power, not wall power. +7. `power-overview.json` — **Homelab Power**: all physical hosts' known CPU + GPU power, with a + stacked-by-host area chart and a per-host line chart. The UPS load panel is whole-rack only; this + is still silicon power, not per-host wall power. ## Kubernetes diff --git a/hosts/12kingdoms-youko/homelab-services/grafana/dashboards/homelab/power-overview.json b/hosts/12kingdoms-youko/homelab-services/grafana/dashboards/homelab/power-overview.json new file mode 100644 index 00000000..ceb1b156 --- /dev/null +++ b/hosts/12kingdoms-youko/homelab-services/grafana/dashboards/homelab/power-overview.json @@ -0,0 +1,872 @@ +{ + "annotations": { + "list": [] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "links": [ + { + "asDropdown": false, + "icon": "external link", + "includeVars": false, + "keepTime": true, + "tags": [], + "targetBlank": false, + "title": "Host detail", + "tooltip": "CPU core/uncore, per-card GPU and energy range for one selected host", + "type": "link", + "url": "/d/homelab-power/host-power-detail" + } + ], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "victoriametrics-metrics-datasource", + "uid": "${DS_PROMETHEUS}" + }, + "description": "CPU package + GPU power. Not wall power.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 1, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "watt" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 0, + "y": 0 + }, + "id": 1, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "targets": [ + { + "datasource": { + "type": "victoriametrics-metrics-datasource", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum(sum by (host) (rate(node_rapl_package_joules_total{path=~\".*intel-rapl:[0-9]+\"}[$__rate_interval]) or (node_hwmon_power_average_watt * on (chip) group_left (chip_name) node_hwmon_chip_names{chip_name=\"amdgpu\"}) or nvidia_smi_power_draw_watts))", + "legendFormat": "", + "range": false, + "instant": true, + "refId": "A" + } + ], + "title": "Known silicon power", + "type": "stat" + }, + { + "datasource": { + "type": "victoriametrics-metrics-datasource", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Physical hosts currently exporting CPU or GPU power metrics.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 8, + "y": 0 + }, + "id": 2, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "targets": [ + { + "datasource": { + "type": "victoriametrics-metrics-datasource", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "count(count by (host) (sum by (host) (rate(node_rapl_package_joules_total{path=~\".*intel-rapl:[0-9]+\"}[$__rate_interval]) or (node_hwmon_power_average_watt * on (chip) group_left (chip_name) node_hwmon_chip_names{chip_name=\"amdgpu\"}) or nvidia_smi_power_draw_watts)))", + "legendFormat": "", + "range": false, + "instant": true, + "refId": "A" + } + ], + "title": "Hosts reporting", + "type": "stat" + }, + { + "datasource": { + "type": "victoriametrics-metrics-datasource", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Whole-rack UPS load percentage; not per-host wall power.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 5, + "x": 14, + "y": 0 + }, + "id": 3, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "targets": [ + { + "datasource": { + "type": "victoriametrics-metrics-datasource", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "max(network_ups_tools_ups_load)", + "legendFormat": "", + "range": false, + "instant": true, + "refId": "A" + } + ], + "title": "UPS total load", + "type": "stat" + }, + { + "datasource": { + "type": "victoriametrics-metrics-datasource", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Sum of RAPL package power across reporting hosts.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 1, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "watt" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 5, + "x": 19, + "y": 0 + }, + "id": 4, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "targets": [ + { + "datasource": { + "type": "victoriametrics-metrics-datasource", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum(sum by (host) (rate(node_rapl_package_joules_total{path=~\".*intel-rapl:[0-9]+\"}[$__rate_interval])))", + "legendFormat": "", + "range": false, + "instant": true, + "refId": "A" + } + ], + "title": "CPU package total", + "type": "stat" + }, + { + "datasource": { + "type": "victoriametrics-metrics-datasource", + "uid": "${DS_PROMETHEUS}" + }, + "description": "CPU package + GPU power; stacked area. This is silicon power, not wall power.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 45, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "watt" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 5 + }, + "id": 10, + "options": { + "legend": { + "calcs": ["lastNotNull", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "victoriametrics-metrics-datasource", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum by (host) (rate(node_rapl_package_joules_total{path=~\".*intel-rapl:[0-9]+\"}[$__rate_interval]) or (node_hwmon_power_average_watt * on (chip) group_left (chip_name) node_hwmon_chip_names{chip_name=\"amdgpu\"}) or nvidia_smi_power_draw_watts)", + "legendFormat": "{{host}}", + "range": true, + "instant": false, + "refId": "A" + } + ], + "title": "Known silicon power by host — stacked", + "type": "timeseries" + }, + { + "datasource": { + "type": "victoriametrics-metrics-datasource", + "uid": "${DS_PROMETHEUS}" + }, + "description": "One line per host, same data as the stacked view.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 8, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "watt" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 14 + }, + "id": 11, + "options": { + "legend": { + "calcs": ["lastNotNull", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "victoriametrics-metrics-datasource", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum by (host) (rate(node_rapl_package_joules_total{path=~\".*intel-rapl:[0-9]+\"}[$__rate_interval]) or (node_hwmon_power_average_watt * on (chip) group_left (chip_name) node_hwmon_chip_names{chip_name=\"amdgpu\"}) or nvidia_smi_power_draw_watts)", + "legendFormat": "{{host}}", + "range": true, + "instant": false, + "refId": "A" + } + ], + "title": "Known silicon power by host — lines", + "type": "timeseries" + }, + { + "datasource": { + "type": "victoriametrics-metrics-datasource", + "uid": "${DS_PROMETHEUS}" + }, + "description": "RAPL package power, one line per host.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 8, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "watt" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 23 + }, + "id": 12, + "options": { + "legend": { + "calcs": ["lastNotNull", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "victoriametrics-metrics-datasource", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum by (host) (rate(node_rapl_package_joules_total{path=~\".*intel-rapl:[0-9]+\"}[$__rate_interval]))", + "legendFormat": "{{host}}", + "range": true, + "instant": false, + "refId": "A" + } + ], + "title": "CPU package power by host", + "type": "timeseries" + }, + { + "datasource": { + "type": "victoriametrics-metrics-datasource", + "uid": "${DS_PROMETHEUS}" + }, + "description": "amdgpu hwmon power and NVIDIA nvidia-smi power, one line per host.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 8, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "watt" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 32 + }, + "id": 13, + "options": { + "legend": { + "calcs": ["lastNotNull", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "victoriametrics-metrics-datasource", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum by (host) (node_hwmon_power_average_watt * on (chip) group_left (chip_name) node_hwmon_chip_names{chip_name=\"amdgpu\"}) or sum by (host) (nvidia_smi_power_draw_watts)", + "legendFormat": "{{host}}", + "range": true, + "instant": false, + "refId": "A" + } + ], + "title": "GPU power by host", + "type": "timeseries" + }, + { + "datasource": { + "type": "victoriametrics-metrics-datasource", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Whole-rack UPS load percentage from the NUT exporter. The UPS does not expose real power via this driver.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 41 + }, + "id": 14, + "options": { + "legend": { + "calcs": ["lastNotNull", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "victoriametrics-metrics-datasource", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "max(network_ups_tools_ups_load)", + "legendFormat": "UPS load", + "range": true, + "instant": false, + "refId": "A" + } + ], + "title": "UPS total load (whole rack)", + "type": "timeseries" + }, + { + "datasource": { + "type": "victoriametrics-metrics-datasource", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Current per-host known silicon power. This does not include PSU, VRM, RAM, fans, NVMe or board losses.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "watt" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 41 + }, + "id": 15, + "options": { + "showHeader": true + }, + "targets": [ + { + "datasource": { + "type": "victoriametrics-metrics-datasource", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum by (host) (rate(node_rapl_package_joules_total{path=~\".*intel-rapl:[0-9]+\"}[$__rate_interval]) or (node_hwmon_power_average_watt * on (chip) group_left (chip_name) node_hwmon_chip_names{chip_name=\"amdgpu\"}) or nvidia_smi_power_draw_watts)", + "legendFormat": "", + "range": false, + "instant": true, + "refId": "A" + } + ], + "title": "Current known power per host", + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true + }, + "renameByName": { + "host": "Host", + "Value": "Known power (W)" + } + } + } + ], + "type": "table" + }, + { + "datasource": { + "type": "victoriametrics-metrics-datasource", + "uid": "${DS_PROMETHEUS}" + }, + "gridPos": { + "h": 5, + "w": 24, + "x": 0, + "y": 49 + }, + "id": 20, + "options": { + "code": { + "language": "markdown", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "### What the numbers mean\n\n- **CPU package** comes from RAPL (`node_rapl_package_joules_total`). The counter's rate is watts.\n- **GPU** comes from `node_hwmon_power_average_watt` (amdgpu) and `nvidia_smi_power_draw_watts` (NVIDIA).\n- This dashboard shows **silicon power**, not wall power. It excludes PSU/VRM losses, RAM, fans, NVMe, NIC, RGB and board overhead.\n- The UPS panel is the only true whole-rack number, but this UPS driver only reports **load percentage**, not real power, and it includes non-server devices too.\n\nTo see actual wall watts per machine, add a metered smart plug/PDU per host and export its power metric. Then this dashboard can add a true wall-power panel without changing the per-host workload metrics." + }, + "title": "Metric sources and limitations", + "type": "text" + } + ], + "preload": false, + "refresh": "30s", + "schemaVersion": 41, + "tags": ["homelab", "power", "rapl"], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "default", + "value": "default" + }, + "hide": 0, + "includeAll": false, + "label": "Data source", + "multi": false, + "name": "DS_PROMETHEUS", + "options": [], + "query": "victoriametrics-metrics-datasource", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + } + ] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "Homelab Power", + "uid": "homelab-power-overview", + "version": 1, + "weekStart": "" +} diff --git a/hosts/12kingdoms-youko/homelab-services/grafana/dashboards/homelab/power.json b/hosts/12kingdoms-youko/homelab-services/grafana/dashboards/homelab/power.json new file mode 100644 index 00000000..cba62994 --- /dev/null +++ b/hosts/12kingdoms-youko/homelab-services/grafana/dashboards/homelab/power.json @@ -0,0 +1,606 @@ +{ + "__inputs": [], + "__elements": {}, + "__requires": [], + "annotations": { + "list": [] + }, + "editable": false, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "links": [ + { + "asDropdown": false, + "icon": "external link", + "includeVars": false, + "keepTime": true, + "tags": [], + "targetBlank": false, + "title": "Homelab Power overview", + "tooltip": "Stacked and per-host power across all physical hosts", + "type": "link", + "url": "/d/homelab-power-overview/homelab-power" + } + ], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "panels": [], + "title": "Current power", + "type": "row" + }, + { + "datasource": { + "type": "victoriametrics-metrics-datasource", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Intel RAPL package power, derived from the energy counter rate.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 2, + "unit": "watt", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 30 + }, + { + "color": "red", + "value": 100 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 8, + "x": 0, + "y": 1 + }, + "id": 2, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "targets": [ + { + "datasource": { + "type": "victoriametrics-metrics-datasource", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(node_rapl_package_joules_total{host=\"$host\",path=\"/sys/class/powercap/intel-rapl:0\"}[$__rate_interval]))", + "instant": true, + "range": false, + "legendFormat": "CPU package", + "refId": "A" + } + ], + "title": "CPU package", + "type": "stat" + }, + { + "datasource": { + "type": "victoriametrics-metrics-datasource", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Sum of both R9700 hwmon power sensors.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 2, + "unit": "watt", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 30 + }, + { + "color": "red", + "value": 100 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 8, + "x": 8, + "y": 1 + }, + "id": 3, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "targets": [ + { + "datasource": { + "type": "victoriametrics-metrics-datasource", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(node_hwmon_power_average_watt{host=\"$host\"} * on(chip) group_left(chip_name) node_hwmon_chip_names{host=\"$host\",chip_name=\"amdgpu\"})", + "instant": true, + "range": false, + "legendFormat": "GPU total", + "refId": "A" + } + ], + "title": "GPU total", + "type": "stat" + }, + { + "datasource": { + "type": "victoriametrics-metrics-datasource", + "uid": "${DS_PROMETHEUS}" + }, + "description": "CPU package plus GPU hwmon power. Does not include PSU, VRM, RAM, fans or board losses.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 2, + "unit": "watt", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 30 + }, + { + "color": "red", + "value": 100 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 8, + "x": 16, + "y": 1 + }, + "id": 4, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "targets": [ + { + "datasource": { + "type": "victoriametrics-metrics-datasource", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "(sum(rate(node_rapl_package_joules_total{host=\"$host\",path=\"/sys/class/powercap/intel-rapl:0\"}[$__rate_interval]))) + (sum(node_hwmon_power_average_watt{host=\"$host\"} * on(chip) group_left(chip_name) node_hwmon_chip_names{host=\"$host\",chip_name=\"amdgpu\"}))", + "instant": true, + "range": false, + "legendFormat": "CPU + GPU silicon", + "refId": "A" + } + ], + "title": "CPU + GPU silicon", + "type": "stat" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 5 + }, + "id": 5, + "panels": [], + "title": "CPU RAPL domains", + "type": "row" + }, + { + "datasource": { + "type": "victoriametrics-metrics-datasource", + "uid": "${DS_PROMETHEUS}" + }, + "description": "node_rapl_package_joules_total: base time series is a counter; the dashboard plots its rate in watts.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "showPoints": "never" + }, + "min": 0, + "unit": "watt" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 6 + }, + "id": 6, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "victoriametrics-metrics-datasource", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(node_rapl_package_joules_total{host=\"$host\",path=\"/sys/class/powercap/intel-rapl:0\"}[$__rate_interval]))", + "instant": false, + "range": true, + "legendFormat": "package", + "refId": "A" + } + ], + "title": "CPU package power", + "type": "timeseries" + }, + { + "datasource": { + "type": "victoriametrics-metrics-datasource", + "uid": "${DS_PROMETHEUS}" + }, + "description": "The package total should approximately cover core + uncore plus other package domains.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "showPoints": "never" + }, + "min": 0, + "unit": "watt" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 6 + }, + "id": 7, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "victoriametrics-metrics-datasource", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(node_rapl_core_joules_total{host=\"$host\"}[$__rate_interval]))", + "instant": false, + "range": true, + "legendFormat": "core", + "refId": "A" + }, + { + "datasource": { + "type": "victoriametrics-metrics-datasource", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(node_rapl_uncore_joules_total{host=\"$host\"}[$__rate_interval]))", + "instant": false, + "range": true, + "legendFormat": "uncore", + "refId": "B" + }, + { + "datasource": { + "type": "victoriametrics-metrics-datasource", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(node_rapl_package_joules_total{host=\"$host\",path=\"/sys/class/powercap/intel-rapl:0\"}[$__rate_interval]))", + "instant": false, + "range": true, + "legendFormat": "package", + "refId": "C" + } + ], + "title": "CPU core / uncore power", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 14 + }, + "id": 8, + "panels": [], + "title": "GPU hwmon", + "type": "row" + }, + { + "datasource": { + "type": "victoriametrics-metrics-datasource", + "uid": "${DS_PROMETHEUS}" + }, + "description": "node_hwmon_power_average_watt joined to chip_name=amdgpu.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "showPoints": "never" + }, + "min": 0, + "unit": "watt" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 15 + }, + "id": 9, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "victoriametrics-metrics-datasource", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum by (chip) (node_hwmon_power_average_watt{host=\"$host\"} * on(chip) group_left(chip_name) node_hwmon_chip_names{host=\"$host\",chip_name=\"amdgpu\"})", + "instant": false, + "range": true, + "legendFormat": "{{chip}}", + "refId": "A" + } + ], + "title": "GPU power by card", + "type": "timeseries" + }, + { + "datasource": { + "type": "victoriametrics-metrics-datasource", + "uid": "${DS_PROMETHEUS}" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 15 + }, + "id": 10, + "options": { + "content": "### Metrics\n\n- CPU package: `node_rapl_package_joules_total` (rate is watts).\n- CPU domains: `node_rapl_core_joules_total`, `node_rapl_uncore_joules_total`.\n- GPU: `node_hwmon_power_average_watt` joined with `node_hwmon_chip_names{chip_name=\"amdgpu\"}`.\n\nRAPL covers the CPU package; hwmon covers the two R9700s. Neither includes PSU, VRM, RAM, fans or board losses. For wall power, use the metered plug/UPS dashboard.", + "mode": "markdown" + }, + "title": "Metric sources", + "type": "text" + }, + { + "datasource": { + "type": "victoriametrics-metrics-datasource", + "uid": "${DS_PROMETHEUS}" + }, + "description": "CPU package energy consumed over the dashboard time range, converted from RAPL joules to kWh. CPU package only; not whole-machine wall energy.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 2, + "unit": "suffix:kWh", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 8, + "x": 0, + "y": 23 + }, + "id": 11, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "targets": [ + { + "datasource": { + "type": "victoriametrics-metrics-datasource", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "(sum(increase(node_rapl_package_joules_total{host=\"$host\",path=\"/sys/class/powercap/intel-rapl:0\"}[$__range]))) / 3.6e6", + "instant": true, + "range": false, + "legendFormat": "CPU package", + "refId": "A" + } + ], + "title": "CPU package energy (kWh, selected range)", + "type": "stat" + } + ], + "refresh": "30s", + "schemaVersion": 39, + "tags": ["homelab", "power", "rapl"], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "default", + "value": "default" + }, + "hide": 0, + "includeAll": false, + "label": "datasource", + "multi": false, + "name": "DS_PROMETHEUS", + "options": [], + "query": "victoriametrics-metrics-datasource", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "current": { + "selected": true, + "text": "aquamarine", + "value": "aquamarine" + }, + "datasource": { + "type": "victoriametrics-metrics-datasource", + "uid": "${DS_PROMETHEUS}" + }, + "definition": "label_values(node_rapl_package_joules_total, host)", + "hide": 0, + "includeAll": false, + "label": "Host", + "multi": false, + "name": "host", + "options": [], + "query": { + "query": "label_values(node_rapl_package_joules_total, host)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "sort": 1, + "type": "query" + } + ] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "Host Power (detail)", + "uid": "homelab-power", + "version": 1, + "weekStart": "", + "description": "Detailed power breakdown for one selected RAPL-capable host: CPU package/core/uncore, GPU hwmon where available, and energy ranges. Use Homelab Power for the all-host view." +} diff --git a/hosts/idols-aquamarine/README.md b/hosts/idols-aquamarine/README.md index cd6fcb55..d96bf95f 100644 --- a/hosts/idols-aquamarine/README.md +++ b/hosts/idols-aquamarine/README.md @@ -26,6 +26,46 @@ Two things about this board set the ceiling for the GPU work: R9700 has no xGMI link between cards, and a dual-R9700 report has RCCL deadlocking with vLLM TP=2 even with `NCCL_P2P_DISABLE=1`. See [ROCM.md](./ROCM.md). +## Idle power accounting + +`node_exporter` exports the RAPL package/core/uncore counters and the R9700 `hwmon` power sensors, +so VictoriaMetrics/Grafana can show whether the host is actually idle. For a one-off breakdown: + +```sh +sudo turbostat --quiet --show PkgWatt,RAMWatt,GFXWatt,CorWatt --interval 2 --iterations 5 +sudo powertop +rocm-smi --showpower --showuse --showclocks +``` + +RAPL and `hwmon` only cover silicon domains. They do not include PSU, VRM, fan, NVMe or board +losses; measuring total wall power needs a metering smart plug/PDU or a UPS with per-outlet power. + +### Monitoring overhead + +The scrape is not a meaningful load: node_exporter samples every 30 s, and the measured process cost +is a few CPU ticks per scrape (well below 1 W even after accounting for wakeups). Grafana queries +run on `youko`, not on this host, so opening a dashboard does not wake `aquamarine`'s CPU. Keep the +30 s interval; lowering it would lose useful resolution for no measurable power saving. + +### CPU idle-power settings + +- `hardware-intel-cpu.nix` uses NixOS's native `boot.kernel.sysfs` option to set Intel P-state's + `energy_performance_preference` to `balance_power` on every CPU. This is separate from the + existing x86-wide `powerManagement.cpuFreqGovernor = "powersave"`: the governor chooses the + frequency policy, while EPP tells `intel_pstate` how aggressively to trade power for performance. + A runtime A/B on 2026-11-01 lowered the whole-home meter minimum from 160–170 W to 137 W, though + the reading occasionally returned to the old range. This is a whole-rack number, not a host-only + measurement. +- `processor.max_cstate=2` is intentionally absent so the kernel can use deeper package C-states. If + inference tail latency becomes more important than idle power, restore that kernel parameter; it + was set for decode jitter, not for hardware stability. After the first deployment on 2026-11-01, + `intel_idle` still exposed only `POLL`/`C1`/`C2`/`C3` on BIOS 1013, so deeper CPU/package C-states + still need to be enabled in BIOS before this knob can save more power. +- Safe BIOS-side changes: enable C1E, Intel Speed Shift and CPU/package C-states (limit `Auto` or + `C10`), and enable iGPU Render Standby/RC6. Disable unused Wi-Fi/Bluetooth, onboard audio, SATA, + serial ports and RGB lighting. Do not change the storage mode, and do not enable PCIe ASPM while + the NVMe/GPU stability workarounds are in place. + The iGPU carries display and video work: the kernel binds `i915`, and [hardware-intel-igpu.nix](./hardware-intel-igpu.nix) installs `intel-media-driver` (VA-API), `vpl-gpu-rt` (oneVPL/QSV) and `intel-compute-runtime` (OpenCL/Level-Zero), so transcoding can use diff --git a/hosts/idols-aquamarine/ROCM.md b/hosts/idols-aquamarine/ROCM.md index 68f64c13..e9b6139b 100644 --- a/hosts/idols-aquamarine/ROCM.md +++ b/hosts/idols-aquamarine/ROCM.md @@ -123,9 +123,10 @@ default `0xfff7bfff`; `0xfff7ffff` works). **verified** `amdgpu.ppfeaturemask=0xffffffff` does not address the ATPX/SMU path; it comes from a community tuning repository and no primary source ties it to it. **community-reported** ([r9700-setup](https://github.com/stew675/r9700-setup/blob/HEAD/README.md)). The host's own -`pcie_aspm=off` and `processor.max_cstate=2` are unrelated to this path too — they are set for PCIe -link retraining and C-state exit latency, with their reasons in -[hardware-amd-gpu.nix](./hardware-amd-gpu.nix). +`pcie_aspm=off` is unrelated to this path too — it is set for PCIe link retraining, with its reason +in [hardware-configuration.nix](./hardware-configuration.nix). CPU C-state depth is no longer +capped; the old `processor.max_cstate=2` trade-off was removed for idle power, so re-check inference +tail latency after that change. ## Kernel parameters and module options diff --git a/hosts/idols-aquamarine/default.nix b/hosts/idols-aquamarine/default.nix index f2c91e51..d923952e 100644 --- a/hosts/idols-aquamarine/default.nix +++ b/hosts/idols-aquamarine/default.nix @@ -27,6 +27,7 @@ in # hardware ./hardware-configuration.nix + ./hardware-intel-cpu.nix ./hardware-amd-gpu.nix ./hardware-intel-igpu.nix diff --git a/hosts/idols-aquamarine/hardware-amd-gpu.nix b/hosts/idols-aquamarine/hardware-amd-gpu.nix index b980396a..ce1ab28a 100644 --- a/hosts/idols-aquamarine/hardware-amd-gpu.nix +++ b/hosts/idols-aquamarine/hardware-amd-gpu.nix @@ -36,26 +36,8 @@ # metal a hung card stays hung until the next reboot. Nobody is here to # power-cycle an unattended inference host, so switch the recovery path on. "amdgpu.gpu_recovery=1" - - # ASPM L0s/L1 on a link that carries PCIe traffic for a compute card buys - # little and is a known source of link retraining and timeouts; this repo - # already falls back to the same knob for NVMe APST problems (WA-008). - "pcie_aspm=off" - - # Keep the CPU out of the deepest C-states. Inference waits on short PCIe and - # memory round-trips, and the exit latency from C6/C7 shows up as decode - # jitter. Costs idle power: drop this line if that matters more than tail - # latency. - "processor.max_cstate=2" ]; - # amdgpu firmware. - hardware.enableRedistributableFirmware = true; - - hardware.graphics = { - enable = true; - }; - # ROCm 10.1 is available: https://rocm.docs.amd.com/en/latest/. # nixpkgs still ships 7.2.x while the upgrade awaits split reviews, LLVM/OpenMP # fixes and compatibility work: https://github.com/NixOS/nixpkgs/pull/573885. @@ -125,5 +107,8 @@ # `[ "systemd" "logind" ]` instead of replacing it: only name the collector # this host adds. Repeating the base entries passes `--collector.systemd` # twice, which node_exporter rejects, and the unit restart-loops. + # + # The CPU/RAPL collector lives in ./hardware-intel-cpu.nix; this file only adds + # the AMD GPU `drm` collector. services.prometheus.exporters.node.enabledCollectors = [ "drm" ]; } diff --git a/hosts/idols-aquamarine/hardware-configuration.nix b/hosts/idols-aquamarine/hardware-configuration.nix index 83f249e6..9f536773 100644 --- a/hosts/idols-aquamarine/hardware-configuration.nix +++ b/hosts/idols-aquamarine/hardware-configuration.nix @@ -21,6 +21,10 @@ # DRAM-less KINGBANK KP260 (WA-008 in WORKAROUNDS.md). This disk (Fanxiang # S790) has shown no APST problem; the cap stays as idle-power insurance. "nvme_core.default_ps_max_latency_us=100000" + + # Keep PCIe ASPM disabled. The R9700 links have retraining/timeout issues, + # and this repo already uses the same knob for NVMe APST problems (WA-008). + "pcie_aspm=off" ]; boot.loader.efi.canTouchEfiVariables = true; @@ -65,6 +69,12 @@ # and swap (including /swap/swapfile bind and swapDevices) are managed by disko (disko-fs.nix). nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux"; + + # CPU microcode and both the Intel iGPU and AMD GPU firmware. + hardware.enableRedistributableFirmware = true; + # Shared graphics stack for the i915 console/video path and the R9700s. + hardware.graphics.enable = true; + powerManagement.cpuFreqGovernor = lib.mkDefault "performance"; # ondemand / powersave / performance hardware.cpu.intel.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware; } diff --git a/hosts/idols-aquamarine/hardware-intel-cpu.nix b/hosts/idols-aquamarine/hardware-intel-cpu.nix new file mode 100644 index 00000000..0fe08ab0 --- /dev/null +++ b/hosts/idols-aquamarine/hardware-intel-cpu.nix @@ -0,0 +1,25 @@ +{ + pkgs, + ... +}: +{ + # Ad-hoc CPU idle-power diagnostics: package power, C-state residency and + # wakeup attribution. Long-term RAPL export lives in the shared + # modules/nixos/base/power-monitoring.nix module. + environment.systemPackages = [ + pkgs.linuxPackages_latest.cpupower + pkgs.linuxPackages_latest.turbostat + pkgs.powertop + ]; + + # `powerManagement.cpuFreqGovernor = "powersave"` on x86 only selects the + # cpufreq governor. Intel P-state has a second knob, EPP, which biases the + # hardware between performance and power before the governor acts. Keep the + # existing powersave governor, but use the power-oriented EPP for this host. + # + # `boot.kernel.sysfs` is NixOS's native, declarative interface for sysfs + # attributes. It waits for the cpufreq attribute and writes it idempotently, + # so no private cpupower systemd service is needed. + boot.kernel.sysfs.devices.system.cpu."cpu[0-9]*".cpufreq.energy_performance_preference = + "balance_power"; +} diff --git a/modules/nixos/base/power-monitoring.nix b/modules/nixos/base/power-monitoring.nix new file mode 100644 index 00000000..6fad1b35 --- /dev/null +++ b/modules/nixos/base/power-monitoring.nix @@ -0,0 +1,58 @@ +{ + config, + lib, + mylib, + pkgs, + ... +}: +let + cfg = config.modules.powerMonitoring; +in +{ + options.modules.powerMonitoring.enable = lib.mkEnableOption '' + Export RAPL CPU energy counters through node_exporter and allow only the + exporter to read them. + + This weakens the kernel's root-only energy_uj permission for local RAPL + side-channel reasons, so it is opt-in and must stay disabled on desktops. + ''; + + config = + lib.mkIf (cfg.enable && config.modules.hardwareTools.enable && pkgs.stdenv.hostPlatform.isx86) + { + services.prometheus.exporters.node.enabledCollectors = [ "rapl" ]; + + # Only node_exporter needs the counters. Do not grant the interactive + # user or the world read access: both would expose a local RAPL + # side-channel to desktop sessions and unprivileged processes. + users.groups.rapl = lib.mkIf config.services.prometheus.exporters.node.enable { }; + users.users = lib.mkIf config.services.prometheus.exporters.node.enable { + node-exporter.extraGroups = [ "rapl" ]; + }; + + systemd.services.rapl-permissions = lib.mkIf config.services.prometheus.exporters.node.enable { + description = "Allow node_exporter to read RAPL energy counters"; + wantedBy = [ "multi-user.target" ]; + unitConfig.ConditionPathExists = "/sys/class/powercap"; + serviceConfig = mylib.systemdHardening.privilegedOneshot // { + Type = "oneshot"; + RemainAfterExit = true; + CapabilityBoundingSet = [ "CAP_CHOWN" ]; + }; + script = '' + shopt -s nullglob + for f in /sys/class/powercap/*/energy_uj; do + chgrp rapl "$f" + chmod 0440 "$f" + done + ''; + }; + + systemd.services.prometheus-node-exporter = + lib.mkIf config.services.prometheus.exporters.node.enable + { + requires = [ "rapl-permissions.service" ]; + after = [ "rapl-permissions.service" ]; + }; + }; +} diff --git a/modules/nixos/server/server.nix b/modules/nixos/server/server.nix index c9434748..9fb88868 100644 --- a/modules/nixos/server/server.nix +++ b/modules/nixos/server/server.nix @@ -5,6 +5,10 @@ ../../base ]; + # RAPL is a local side channel, so exposure is opt-in and scoped to the + # server role. Desktops keep the kernel-default root-only energy_uj. + modules.powerMonitoring.enable = true; + # Servers used to keep the firewall off, trusting the router alone. They now # run the shared base firewall (modules/nixos/base/networking/firewall.nix) # like every other host, as defence in depth behind the router. diff --git a/outputs/x86_64-linux/tests/cpu-power/expected.nix b/outputs/x86_64-linux/tests/cpu-power/expected.nix index e8ebcd19..8eea5079 100644 --- a/outputs/x86_64-linux/tests/cpu-power/expected.nix +++ b/outputs/x86_64-linux/tests/cpu-power/expected.nix @@ -8,5 +8,9 @@ noEeeWorkaround = true; noFrequencyCaps = true; guestsHaveNoCpuPolicy = true; + serverRaplEnabled = lib.genAttrs [ "aquamarine" "shoryu" "shushou" "youko" ] (_: true); + serverRaplReaderOnly = true; + desktopRaplDisabled = true; + guestsHaveNoRapl = true; desktopBatteryReporting = true; } diff --git a/outputs/x86_64-linux/tests/cpu-power/expr.nix b/outputs/x86_64-linux/tests/cpu-power/expr.nix index 1e66de4a..ab424a73 100644 --- a/outputs/x86_64-linux/tests/cpu-power/expr.nix +++ b/outputs/x86_64-linux/tests/cpu-power/expr.nix @@ -12,6 +12,12 @@ let "shushou" "youko" ]; + serverHosts = [ + "aquamarine" + "shoryu" + "shushou" + "youko" + ]; in { governors = lib.genAttrs physicalHosts ( @@ -31,5 +37,23 @@ in guestsHaveNoCpuPolicy = lib.all (name: !(configs.${name}.config.systemd.services ? cpufreq)) ( lib.subtractLists physicalHosts (builtins.attrNames configs) ); + serverRaplEnabled = lib.genAttrs serverHosts ( + name: + configs.${name}.config.modules.powerMonitoring.enable + && builtins.elem "rapl" configs.${name}.config.services.prometheus.exporters.node.enabledCollectors + && configs.${name}.config.systemd.services ? rapl-permissions + ); + serverRaplReaderOnly = lib.all ( + name: + builtins.elem "rapl" configs.${name}.config.users.users.node-exporter.extraGroups + && !(builtins.elem "rapl" (configs.${name}.config.users.users.ryan.extraGroups or [ ])) + ) serverHosts; + desktopRaplDisabled = + !configs.ai-niri.config.modules.powerMonitoring.enable + && !(builtins.elem "rapl" configs.ai-niri.config.services.prometheus.exporters.node.enabledCollectors) + && !(configs.ai-niri.config.systemd.services ? rapl-permissions); + guestsHaveNoRapl = lib.all (name: !(configs.${name}.config.systemd.services ? rapl-permissions)) ( + lib.subtractLists physicalHosts (builtins.attrNames configs) + ); desktopBatteryReporting = configs.ai-niri.config.services.upower.enable; } diff --git a/outputs/x86_64-linux/tests/idols-aquamarine/expected.nix b/outputs/x86_64-linux/tests/idols-aquamarine/expected.nix index 852116d4..947dea93 100644 --- a/outputs/x86_64-linux/tests/idols-aquamarine/expected.nix +++ b/outputs/x86_64-linux/tests/idols-aquamarine/expected.nix @@ -5,7 +5,8 @@ runtimePmDisabled = true; gpuRecoveryEnabled = true; aspmDisabled = true; - cStatesCapped = true; + deepCStatesAllowed = true; + cpuEppConfigured = true; nativeIpv6Preference = true; sourceCheckoutsPersistent = true; cachePersistent = true; @@ -17,6 +18,8 @@ igpuMediaStackInstalled = true; openvinoGpuWrapperInstalled = true; gpuMetricsExported = true; + raplMetricsExported = true; + raplPermissionsExposed = true; collectorFlagsUnique = true; inferenceUserCanRender = true; noHomeManager = true; diff --git a/outputs/x86_64-linux/tests/idols-aquamarine/expr.nix b/outputs/x86_64-linux/tests/idols-aquamarine/expr.nix index 8b24b368..4399a37f 100644 --- a/outputs/x86_64-linux/tests/idols-aquamarine/expr.nix +++ b/outputs/x86_64-linux/tests/idols-aquamarine/expr.nix @@ -23,7 +23,16 @@ in # An unattended host recovers a hung card itself instead of waiting for a reboot. gpuRecoveryEnabled = builtins.elem "amdgpu.gpu_recovery=1" cfg.boot.kernelParams; aspmDisabled = builtins.elem "pcie_aspm=off" cfg.boot.kernelParams; - cStatesCapped = builtins.elem "processor.max_cstate=2" cfg.boot.kernelParams; + deepCStatesAllowed = !(builtins.elem "processor.max_cstate=2" cfg.boot.kernelParams); + cpuEppConfigured = + lib.attrByPath [ + "devices" + "system" + "cpu" + "cpu[0-9]*" + "cpufreq" + "energy_performance_preference" + ] null cfg.boot.kernel.sysfs == "balance_power"; # Aqua uses native IPv6; the gateway returns real AAAA records instead of # requiring a host-local IPv4 precedence workaround. @@ -73,6 +82,8 @@ in package: (package.name or "") == "openvino-gpu" ) cfg.environment.systemPackages; gpuMetricsExported = builtins.elem "drm" cfg.services.prometheus.exporters.node.enabledCollectors; + raplMetricsExported = builtins.elem "rapl" cfg.services.prometheus.exporters.node.enabledCollectors; + raplPermissionsExposed = cfg.systemd.services ? rapl-permissions; # node_exporter rejects a repeated flag, and this list merges with the base # module's entries, so a duplicate restart-loops the unit. collectorFlagsUnique = diff --git a/outputs/x86_64-linux/tests/security-service-hardening/expected.nix b/outputs/x86_64-linux/tests/security-service-hardening/expected.nix index 1bca34e2..0f2103d5 100644 --- a/outputs/x86_64-linux/tests/security-service-hardening/expected.nix +++ b/outputs/x86_64-linux/tests/security-service-hardening/expected.nix @@ -34,6 +34,9 @@ in }; privilegedOneshot = { + rapl-permissions = privilegedOneshot // { + CapabilityBoundingSet = [ "CAP_CHOWN" ]; + }; amdgpu-power-cap = privilegedOneshot; hdd-no-spindown = privilegedOneshot // { CapabilityBoundingSet = [ "CAP_SYS_RAWIO" ]; diff --git a/outputs/x86_64-linux/tests/security-service-hardening/expr.nix b/outputs/x86_64-linux/tests/security-service-hardening/expr.nix index fbd43600..429011e1 100644 --- a/outputs/x86_64-linux/tests/security-service-hardening/expr.nix +++ b/outputs/x86_64-linux/tests/security-service-hardening/expr.nix @@ -38,6 +38,7 @@ units "youko" [ // units "aquamarine" [ "whisper-asr" ] // { privilegedOneshot = { + rapl-permissions = privilegedUnit "aquamarine" "rapl-permissions"; amdgpu-power-cap = privilegedUnit "aquamarine" "amdgpu-power-cap"; hdd-no-spindown = privilegedUnit "youko" "hdd-no-spindown"; };