starred/nix-config-ryan4yin
1
0 Fork 0
mirror of https://github.com/ryan4yin/nix-config.git synced 2026-11-13 12:13:36 +01:00
Code Issues Packages Projects Releases 10 Wiki Activity

feat(monitoring): GPU-aware btop variants and GPU metrics (#330)

Browse Source
This commit is contained in:
ryan4yin | 二花 authored and GitHub committed 2026-09-30 00:16:50 +08:00
1 parent 523f4940a7
commit 9ab5eca60c
14 files changed
+690 -10
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

home/base/core/btop.nix
+13 -1
View File
@@ -1,9 +1,21 @@
{
pkgs,
...
}:
{
# replacement of htop/nmon
programs.btop = {
enable = true;
# On NixOS `modules.btop` installs the single GPU-aware build (or a
# CAP_PERFMON wrapper), so don't let Home Manager add a second, plain btop.
# macOS has no `modules.btop` and takes btop from Home Manager.
package = if pkgs.stdenv.hostPlatform.isDarwin then pkgs.btop else null;
settings = {
theme_background = false; # make btop transparent
# Use the terminal background so btop is transparent. No visible effect
# inside zellij yet: its panes have been opaque since 0.44
# (https://github.com/zellij-org/zellij/issues/5175), so this only shows
# when btop runs directly in a translucent terminal.
theme_background = false;
};
};
}
hosts/12kingdoms-shoryu/default.nix
+5
View File
@@ -26,6 +26,11 @@ in
modules.btrbk.enable = true;
# MoreFine S500Plus: Radeon iGPU. The ROCm build of btop shows the GPU panel;
# node_exporter's drm collector exposes its utilization/VRAM.
modules.btop.package = pkgs.btop-rocm;
services.prometheus.exporters.node.enabledCollectors = [ "drm" ];
boot.kernelParams = [
# Use transparent huge pages on demand (madvise) instead of a fixed 1G hugetlb
# pool. The static pool cannot be overcommitted or shared with the host /
hosts/12kingdoms-shushou/default.nix
+5
View File
@@ -25,6 +25,11 @@ in
modules.btrbk.enable = true;
# MinisForum UM560: Radeon iGPU. The ROCm build of btop shows the GPU panel;
# node_exporter's drm collector exposes its utilization/VRAM.
modules.btop.package = pkgs.btop-rocm;
services.prometheus.exporters.node.enabledCollectors = [ "drm" ];
boot.kernelParams = [
# Use transparent huge pages on demand (madvise) instead of a fixed 1G hugetlb
# pool. The static pool cannot be overcommitted or shared with the host /
hosts/12kingdoms-youko/default.nix
+5
View File
@@ -59,6 +59,11 @@ in
modules.btrbk.enable = true;
# Beelink GTR5: Radeon iGPU. The ROCm build of btop shows the GPU panel;
# node_exporter's drm collector exposes its utilization/VRAM.
modules.btop.package = pkgs.btop-rocm;
services.prometheus.exporters.node.enabledCollectors = [ "drm" ];
# The restic REST server the desktops push their backups to. Private repos,
# so each client only reaches the repository named after its user. Credentials
# come from agenix; caddy terminates TLS in front of it.
hosts/12kingdoms-youko/homelab-services.nix
-4
View File
@@ -6,10 +6,6 @@
# scanPaths. Its HDD disko config only defines the two data disks (no root
# disk), so it is imported as-is -- disko's NixOS module only generates the
# fileSystems/mounts, it never reformats.
#
# keep aquamarine's IP on this host's bridge so its services and DNS keep
# resolving after the migration (the NixOS list option merges with the .183)
systemd.network.networks."10-br0".networkConfig.Address = [ "192.168.5.101/24" ];
# the same secret groups the aqua VM enabled
modules.secrets.server = {
hosts/12kingdoms-youko/homelab-services/grafana/dashboards/homelab/gpu.json
+567
View File
@@ -0,0 +1,567 @@
{
"__inputs": [],
"__elements": {},
"__requires": [
{
"type": "grafana",
"id": "grafana",
"name": "Grafana",
"version": "11.2.0"
},
{
"type": "datasource",
"id": "prometheus",
"name": "Prometheus",
"version": "1.0.0"
},
{
"type": "panel",
"id": "timeseries",
"name": "Time series",
"version": ""
}
],
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": {
"type": "datasource",
"uid": "grafana"
},
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"type": "dashboard"
}
]
},
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 1,
"id": null,
"links": [],
"panels": [
{
"collapsed": false,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 0
},
"id": 100,
"panels": [],
"title": "AMD Radeon (amdgpu, node_exporter)",
"type": "row"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"description": "node_drm_gpu_busy_percent (node_exporter drm collector)",
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"drawStyle": "line",
"fillOpacity": 10,
"lineWidth": 1,
"showPoints": "never"
},
"min": 0,
"max": 100,
"unit": "percent"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 8,
"x": 0,
"y": 1
},
"id": 1,
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "node_drm_gpu_busy_percent{host=\"$host\"}",
"legendFormat": "{{host}} {{card}}",
"refId": "A"
}
],
"title": "GPU busy",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"description": "node_drm_memory_vram_used_bytes / node_drm_memory_vram_size_bytes",
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"drawStyle": "line",
"fillOpacity": 10,
"lineWidth": 1,
"showPoints": "never"
},
"unit": "bytes"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 8,
"x": 8,
"y": 1
},
"id": 2,
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "node_drm_memory_vram_used_bytes{host=\"$host\"}",
"legendFormat": "{{host}} {{card}} used",
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "node_drm_memory_vram_size_bytes{host=\"$host\"}",
"legendFormat": "{{host}} {{card}} size",
"refId": "B"
}
],
"title": "VRAM",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"description": "node_hwmon_power_watt / node_hwmon_temp_celsius for the amdgpu chip",
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"drawStyle": "line",
"fillOpacity": 10,
"lineWidth": 1,
"showPoints": "never"
}
},
"overrides": [
{
"matcher": {
"id": "byName",
"options": "power"
},
"properties": [
{
"id": "unit",
"value": "watt"
}
]
},
{
"matcher": {
"id": "byName",
"options": "temp"
},
"properties": [
{
"id": "unit",
"value": "celsius"
}
]
}
]
},
"gridPos": {
"h": 8,
"w": 8,
"x": 16,
"y": 1
},
"id": 3,
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "node_hwmon_power_watt{host=\"$host\", chip=~\"$chip\"}",
"legendFormat": "power",
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "node_hwmon_temp_celsius{host=\"$host\", chip=~\"$chip\"}",
"legendFormat": "temp",
"refId": "B"
}
],
"title": "Power / temperature",
"type": "timeseries"
},
{
"collapsed": false,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 9
},
"id": 101,
"panels": [],
"title": "NVIDIA (nvidia_gpu_exporter)",
"type": "row"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"description": "nvidia_smi_utilization_gpu_ratio * 100",
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"drawStyle": "line",
"fillOpacity": 10,
"lineWidth": 1,
"showPoints": "never"
},
"min": 0,
"max": 100,
"unit": "percent"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 8,
"x": 0,
"y": 10
},
"id": 4,
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "nvidia_smi_utilization_gpu_ratio{host=\"$host\"} * 100 * on(uuid) group_left(name) nvidia_smi_gpu_info{host=\"$host\"}",
"legendFormat": "{{name}} gpu",
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "nvidia_smi_utilization_memory_ratio{host=\"$host\"} * 100 * on(uuid) group_left(name) nvidia_smi_gpu_info{host=\"$host\"}",
"legendFormat": "{{name}} mem",
"refId": "B"
}
],
"title": "GPU utilization",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"description": "nvidia_smi_memory_used_bytes / nvidia_smi_memory_total_bytes",
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"drawStyle": "line",
"fillOpacity": 10,
"lineWidth": 1,
"showPoints": "never"
},
"unit": "bytes"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 8,
"x": 8,
"y": 10
},
"id": 5,
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "nvidia_smi_memory_used_bytes{host=\"$host\"} * on(uuid) group_left(name) nvidia_smi_gpu_info{host=\"$host\"}",
"legendFormat": "{{name}} used",
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "nvidia_smi_memory_total_bytes{host=\"$host\"} * on(uuid) group_left(name) nvidia_smi_gpu_info{host=\"$host\"}",
"legendFormat": "{{name}} total",
"refId": "B"
}
],
"title": "Memory",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"description": "nvidia_smi_temperature_gpu / nvidia_smi_power_draw_watts",
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"drawStyle": "line",
"fillOpacity": 10,
"lineWidth": 1,
"showPoints": "never"
}
},
"overrides": [
{
"matcher": {
"id": "byName",
"options": "power"
},
"properties": [
{
"id": "unit",
"value": "watt"
}
]
},
{
"matcher": {
"id": "byName",
"options": "temp"
},
"properties": [
{
"id": "unit",
"value": "celsius"
}
]
}
]
},
"gridPos": {
"h": 8,
"w": 8,
"x": 16,
"y": 10
},
"id": 6,
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "nvidia_smi_temperature_gpu{host=\"$host\"} * on(uuid) group_left(name) nvidia_smi_gpu_info{host=\"$host\"}",
"legendFormat": "temp",
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "nvidia_smi_power_draw_watts{host=\"$host\"} * on(uuid) group_left(name) nvidia_smi_gpu_info{host=\"$host\"}",
"legendFormat": "power",
"refId": "B"
}
],
"title": "Power / temperature",
"type": "timeseries"
}
],
"refresh": "30s",
"schemaVersion": 39,
"tags": ["homelab", "gpu"],
"templating": {
"list": [
{
"current": {
"selected": false,
"text": "default",
"value": "default"
},
"hide": 0,
"includeAll": false,
"label": "datasource",
"multi": false,
"name": "DS_PROMETHEUS",
"options": [],
"query": "prometheus",
"refresh": 1,
"regex": "",
"skipUrlSync": false,
"type": "datasource"
},
{
"current": {},
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"definition": "query_result(count by (host) (node_drm_gpu_busy_percent or nvidia_smi_gpu_info))",
"includeAll": false,
"label": "Host",
"multi": false,
"name": "host",
"options": [],
"query": {
"qryType": 1,
"query": "query_result(count by (host) (node_drm_gpu_busy_percent or nvidia_smi_gpu_info))",
"refId": "PrometheusVariableQueryEditor-VariableQuery"
},
"refresh": 1,
"regex": "/host=\"([^\"]+)\"/",
"sort": 1,
"type": "query"
},
{
"current": {},
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"definition": "label_values(node_hwmon_chip_names{chip_name=\"amdgpu\"}, chip)",
"includeAll": true,
"label": "amdgpu chip",
"multi": false,
"name": "chip",
"options": [],
"query": {
"qryType": 1,
"query": "label_values(node_hwmon_chip_names{chip_name=\"amdgpu\"}, chip)",
"refId": "PrometheusVariableQueryEditor-VariableQuery"
},
"refresh": 1,
"regex": "",
"sort": 1,
"type": "query"
}
]
},
"time": {
"from": "now-6h",
"to": "now"
},
"timepicker": {},
"timezone": "browser",
"title": "GPU",
"uid": "gpu-homelab",
"version": 1,
"weekStart": ""
}
hosts/12kingdoms-youko/homelab-services/monitoring/victoriametrics.nix
+23 -4
View File
@@ -6,7 +6,10 @@
let
# Powered-off hosts (SBCs) plus shoukei (exporter disabled on the
# machine); remove entries when they come back online.
# `aquamarine` is decommissioned: its IP now lives on youko's bridge, so
# scraping it would relabel youko's metrics as aquamarine.
offlineHosts = [
"aquamarine"
"shoukei"
"suzu"
"suzi"
@@ -121,7 +124,7 @@ in
targets = [ "127.0.0.1:9187" ];
labels.type = "app";
labels.app = "postgresql";
labels.host = "aquamarine";
labels.host = "youko";
labels.env = "homelab";
labels.cluster = "homelab";
}
@@ -137,7 +140,7 @@ in
targets = [ "127.0.0.1:10000" ];
labels.type = "app";
labels.app = "sftpgo";
labels.host = "aquamarine";
labels.host = "youko";
labels.env = "homelab";
labels.cluster = "homelab";
}
@@ -152,7 +155,7 @@ in
targets = [ "localhost:9093" ];
labels.type = "app";
labels.app = "alertmanager";
labels.host = "aquamarine";
labels.host = "youko";
labels.env = "homelab";
labels.cluster = "homelab";
}
@@ -168,7 +171,7 @@ in
targets = [ "localhost:9090" ];
labels.type = "app";
labels.app = "victoriametrics";
labels.host = "aquamarine";
labels.host = "youko";
labels.env = "homelab";
labels.cluster = "homelab";
}
@@ -206,6 +209,22 @@ in
}
];
}
{
job_name = "nvidia-gpu-exporter-ai";
scrape_interval = "30s";
metrics_path = "/metrics";
static_configs = [
{
# community NVIDIA exporter on the desktop (nvidia-smi based)
targets = [ "${myvars.networking.hostsAddr.ai.ipv4}:9835" ];
labels.type = "app";
labels.app = "nvidia-gpu";
labels.host = "ai";
labels.env = "homelab";
labels.cluster = "homelab";
}
];
}
]
# --- Hosts --- #
++ (lib.attrsets.foldlAttrs
hosts/idols-ai/hardware-nvidia.nix
+9
View File
@@ -50,6 +50,15 @@
enable32Bit = true;
};
# GPU monitoring: CUDA btop (CAP_PERFMON also lets it read the Intel iGPU),
# nvtop, and the community nvidia_gpu_exporter.
modules.btop = {
package = pkgs.btop-cuda;
perfmon = true;
};
environment.systemPackages = [ pkgs.nvtopPackages.full ];
services.prometheus.exporters.nvidia-gpu.enable = true;
services.sunshine.settings = {
adapter_name = "/dev/dri/by-path/pci-0000:00:02.0-render"; # Intel iGPU
encoder = "vaapi";
modules/base/packages.nix
+2 -1
View File
@@ -21,7 +21,8 @@
# system monitoring
procs # a moreden ps
btop
# btop is provided per host: the GPU-aware `modules.btop` build on NixOS, or
# Home Manager on macOS. Not listed here to avoid a second, plain copy.
# archives
zip
modules/nixos/base/btop.nix
+50
View File
@@ -0,0 +1,50 @@
{
config,
lib,
pkgs,
...
}:
let
cfg = config.modules.btop;
in
{
options.modules.btop = {
package = lib.mkOption {
type = lib.types.package;
default = pkgs.btop;
defaultText = lib.literalExpression "pkgs.btop";
description = ''
btop build to install. The per-vendor GPU modules set this to
`btop-cuda`/`btop-rocm`, whose RUNPATH includes the vendor library that
btop otherwise cannot find (and Intel needs no library, only perf
access, see `perfmon`).
'';
};
perfmon = lib.mkOption {
type = lib.types.bool;
default = false;
description = ''
Install btop as a `CAP_PERFMON` wrapper so it can read Intel GPU stats
without lowering `kernel.perf_event_paranoid` system-wide.
'';
};
};
config = {
# Exactly one btop per host: a plain package normally, or the capability
# wrapper only when `perfmon` is set. Home Manager skips its own copy on
# NixOS (see home/base/core/btop.nix).
environment.systemPackages = lib.optional (!cfg.perfmon) cfg.package;
# perf_event_paranoid stays at its default; only the wrapper gets the
# capability, and exactly one `btop` remains on PATH (/run/wrappers/bin).
security.wrappers = lib.mkIf cfg.perfmon {
btop = {
source = "${cfg.package}/bin/btop";
owner = "root";
group = "root";
capabilities = "cap_perfmon+ep";
};
};
};
}
modules/nixos/base/networking/firewall.nix
+4
View File
@@ -25,6 +25,10 @@ in
# LAN/tailnet/local containers.
ip saddr != ${monitoringHost} tcp dport 9100 drop
# The NVIDIA GPU exporter (desktop only) is likewise scraped only by the
# monitoring host.
ip saddr != ${monitoringHost} tcp dport 9835 drop
# The local container bridge and the tailnet have the same access as the
# LAN. Kept after the drop above so neither can reach node_exporter.
iifname "podman0" accept
modules/nixos/desktop.nix
+5
View File
@@ -24,6 +24,11 @@ in
};
config = mkMerge [
# Mesa userspace (OpenGL/Vulkan) for every GUI host.
{
hardware.graphics.enable = true;
}
(mkIf cfgWayland.enable {
####################################################################
# NixOS's Configuration for Wayland based Window Manager
modules/nixos/server/server-aarch64.nix
+1
View File
@@ -7,6 +7,7 @@
{
imports = [
../base/btrbk.nix
../base/btop.nix
../base/core.nix
../base/i18n.nix
../base/monitoring.nix
modules/nixos/server/server-riscv64.nix
+1
View File
@@ -5,6 +5,7 @@
# =========================================================================
imports = [
../base/btop.nix
../base/core.nix
../base/i18n.nix
../base/monitoring.nix