From dc3ec8c1597ede717e6baf631458488edc1bcccc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?ryan4yin=20=7C=20=E4=BA=8C=E8=8A=B1?= Date: Sat, 3 Oct 2026 11:13:46 +0800 Subject: [PATCH] feat(storage): relax ai NVMe workarounds and add drive SMART monitoring (#359) * fix(ai): relax NVMe power-management workarounds and enable smartd Logs show no NVMe errors, so allow APST again (capped at 100 ms), restore the default io_timeout, and drop pcie_aspm=off. Add smartd for drive health visibility. * feat(storage): add drive/SSD SMART monitoring on physical hosts node_exporter's nvme collector only exposes node_nvme_info and namespace capacity, not SMART health, so SSD wear/lifespan and drive temperature were not observable anywhere. - enable services.prometheus.exporters.smartctl on physical hosts, gated on modules.hardwareTools (off on QEMU/MicroVM guests), bound to the host's LAN address on port 9633 - firewall: drop 9633 from anything but the monitoring host - youko: scrape smartctl- for the physical hosts - alert rules (storage.yml): SMART health, NVMe wear, available spare, media errors, critical warning, and drive temperature (warn 70 / crit 80 C) - Grafana dashboard "Drive SMART (smartctl)" --- .../grafana/dashboards/README.md | 2 + .../grafana/dashboards/homelab/smartctl.json | 584 ++++++++++++++++++ .../monitoring/alert_rules/storage.yml | 104 ++++ .../monitoring/victoriametrics.nix | 34 +- hosts/idols-ai/default.nix | 6 + hosts/idols-ai/hardware-configuration.nix | 37 +- modules/nixos/base/monitoring.nix | 21 + modules/nixos/base/networking/firewall.nix | 4 + 8 files changed, 773 insertions(+), 19 deletions(-) create mode 100644 hosts/12kingdoms-youko/homelab-services/grafana/dashboards/homelab/smartctl.json create mode 100644 hosts/12kingdoms-youko/homelab-services/monitoring/alert_rules/storage.yml diff --git a/hosts/12kingdoms-youko/homelab-services/grafana/dashboards/README.md b/hosts/12kingdoms-youko/homelab-services/grafana/dashboards/README.md index cd6c6adb..1f71040f 100644 --- a/hosts/12kingdoms-youko/homelab-services/grafana/dashboards/README.md +++ b/hosts/12kingdoms-youko/homelab-services/grafana/dashboards/README.md @@ -4,6 +4,8 @@ 1. https://grafana.com/grafana/dashboards/1860-node-exporter-full/ 2. https://grafana.com/grafana/dashboards/9578-alertmanager/ +3. `smartctl.json` — drive/SSD SMART health (wear, spare, media errors, temperature). Custom, built + on the `smartctl_exporter` metrics. ## Kubernetes diff --git a/hosts/12kingdoms-youko/homelab-services/grafana/dashboards/homelab/smartctl.json b/hosts/12kingdoms-youko/homelab-services/grafana/dashboards/homelab/smartctl.json new file mode 100644 index 00000000..fc772172 --- /dev/null +++ b/hosts/12kingdoms-youko/homelab-services/grafana/dashboards/homelab/smartctl.json @@ -0,0 +1,584 @@ +{ + "__inputs": [], + "__elements": {}, + "__requires": [ + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "11.2.0" + }, + { + "type": "datasource", + "id": "prometheus", + "name": "Prometheus", + "version": "1.0.0" + }, + { + "type": "panel", + "id": "stat", + "name": "Stat", + "version": "" + }, + { + "type": "panel", + "id": "timeseries", + "name": "Time series", + "version": "" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": null, + "links": [], + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "mappings": [ + { + "options": { + "0": { + "color": "red", + "text": "FAILED" + }, + "1": { + "color": "green", + "text": "OK" + } + }, + "type": "value" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 0 + }, + "id": 1, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "smartctl_device_smart_status{host=~\"$host\",device=~\"$device\"}", + "instant": true, + "legendFormat": "{{host}} {{device}}", + "refId": "A" + } + ], + "title": "SMART health", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "unit": "h" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 0 + }, + "id": 2, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "smartctl_device_power_on_seconds{host=~\"$host\",device=~\"$device\"} / 3600", + "instant": true, + "legendFormat": "{{host}} {{device}}", + "refId": "A" + } + ], + "title": "Power-on hours", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 0 + }, + "id": 3, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "smartctl_device_media_errors{host=~\"$host\",device=~\"$device\"}", + "instant": true, + "legendFormat": "{{host}} {{device}}", + "refId": "A" + } + ], + "title": "Media errors", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 0 + }, + "id": 4, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "smartctl_device_critical_warning{host=~\"$host\",device=~\"$device\"}", + "instant": true, + "legendFormat": "{{host}} {{device}}", + "refId": "A" + } + ], + "title": "NVMe critical warning", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 2, + "showPoints": "never" + }, + "unit": "celsius" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 4 + }, + "id": 5, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "max by (host, device) (smartctl_device_temperature{host=~\"$host\",device=~\"$device\"})", + "legendFormat": "{{host}} {{device}}", + "refId": "A" + } + ], + "title": "Temperature", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 2, + "showPoints": "never" + }, + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 80 + }, + { + "color": "red", + "value": 90 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 4 + }, + "id": 6, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "max by (host, device) (smartctl_device_percentage_used{host=~\"$host\",device=~\"$device\"})", + "legendFormat": "{{host}} {{device}}", + "refId": "A" + } + ], + "title": "SSD wear (percentage used)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 2, + "showPoints": "never" + }, + "max": 100, + "min": 0, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 12 + }, + "id": 7, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "max by (host, device) (smartctl_device_available_spare{host=~\"$host\",device=~\"$device\"})", + "legendFormat": "{{host}} {{device}} spare", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "max by (host, device) (smartctl_device_available_spare_threshold{host=~\"$host\",device=~\"$device\"})", + "legendFormat": "{{host}} {{device}} threshold", + "refId": "B" + } + ], + "title": "Available spare", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 2, + "showPoints": "never" + }, + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 12 + }, + "id": 8, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "rate(smartctl_device_bytes_written{host=~\"$host\",device=~\"$device\"}[5m])", + "legendFormat": "{{host}} {{device}}", + "refId": "A" + } + ], + "title": "Bytes written (per second)", + "type": "timeseries" + } + ], + "refresh": "1m", + "schemaVersion": 39, + "tags": ["homelab", "smartctl", "storage", "ssd"], + "templating": { + "list": [ + { + "current": {}, + "hide": 0, + "includeAll": false, + "label": "Datasource", + "multi": false, + "name": "DS_PROMETHEUS", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "allValue": ".*", + "current": {}, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "label": "Host", + "multi": true, + "name": "host", + "options": [], + "query": "label_values(smartctl_device, host)", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "allValue": ".*", + "current": {}, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "label": "Device", + "multi": true, + "name": "device", + "options": [], + "query": "label_values(smartctl_device{host=~\"$host\"}, device)", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + } + ] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "Drive SMART (smartctl)", + "uid": "smartctl-homelab", + "version": 1 +} diff --git a/hosts/12kingdoms-youko/homelab-services/monitoring/alert_rules/storage.yml b/hosts/12kingdoms-youko/homelab-services/monitoring/alert_rules/storage.yml new file mode 100644 index 00000000..cd9bed97 --- /dev/null +++ b/hosts/12kingdoms-youko/homelab-services/monitoring/alert_rules/storage.yml @@ -0,0 +1,104 @@ +groups: + - name: storage.rules + rules: + # SMART "overall-health self-assessment" (ATA/SAS) and NVMe health status. + # 0 means the drive failed its own health check. + - alert: DriveHealthFailed + expr: smartctl_device_smart_status == 0 + for: 10m + labels: + severity: critical + annotations: + summary: Drive {{ $labels.device }} on {{ $labels.host }} reports failing SMART health + description: + "{{ $labels.model_name }} (serial {{ $labels.serial_number }}) failed its SMART + overall-health self-assessment." + + # NVMe wear: percentage of the rated write endurance already consumed. + - alert: SSDWearHigh + expr: smartctl_device_percentage_used > 80 + for: 1h + labels: + severity: warning + annotations: + summary: SSD {{ $labels.device }} on {{ $labels.host }} is {{ $value }}% worn + description: + "{{ $labels.model_name }} (serial {{ $labels.serial_number }}) has consumed {{ $value + }}% of its rated write endurance." + + - alert: SSDWearCritical + expr: smartctl_device_percentage_used > 90 + for: 10m + labels: + severity: critical + annotations: + summary: SSD {{ $labels.device }} on {{ $labels.host }} is {{ $value }}% worn + description: + "{{ $labels.model_name }} is nearly out of write endurance; plan a replacement." + + # NVMe available spare fell below the drive's own threshold. + - alert: SSDSpareLow + expr: smartctl_device_available_spare < smartctl_device_available_spare_threshold + for: 10m + labels: + severity: critical + annotations: + summary: SSD {{ $labels.device }} on {{ $labels.host }} spare blocks below threshold + description: + "Available spare {{ $value }}% is below the drive threshold; the controller may + transition to read-only." + + # NVMe critical warning bitfield (spare low, temperature, reliability, + # read-only, volatile-memory backup). + - alert: SSDControllerCriticalWarning + expr: smartctl_device_critical_warning > 0 + for: 10m + labels: + severity: critical + annotations: + summary: SSD {{ $labels.device }} on {{ $labels.host }} has a controller critical warning + description: "smartctl_device_critical_warning = {{ $value }} (non-zero bitfield)." + + # Unrecovered data-integrity errors (uncorrectable ECC, CRC, LBA tag). + - alert: DriveMediaErrors + expr: smartctl_device_media_errors > 0 + for: 10m + labels: + severity: warning + annotations: + summary: Drive {{ $labels.device }} on {{ $labels.host }} has media errors + description: "{{ $value }} unrecovered data-integrity errors recorded." + + # Drive temperature (SSD/HDD). Consumer M.2 NVMe throttles around 70-80 C. + - alert: DriveTemperatureHigh + expr: max by (host, device) (smartctl_device_temperature) > 70 + for: 15m + labels: + severity: warning + annotations: + summary: Drive {{ $labels.device }} on {{ $labels.host }} is hot ({{ $value }} C) + description: "Drive temperature above 70 C; NVMe throttles around 70-80 C." + + - alert: DriveTemperatureCritical + expr: max by (host, device) (smartctl_device_temperature) > 80 + for: 5m + labels: + severity: critical + annotations: + summary: + Drive {{ $labels.device }} on {{ $labels.host }} is critically hot ({{ $value }} C) + description: "Drive temperature above 80 C; risk of throttling or failure." + + # SATA/SAS SSD wear via the ATA Wear_Leveling_Count attribute + # (normalized value: 100 = new, lower = more worn). + - alert: AtaSSDWearHigh + expr: + smartctl_device_attribute{attribute_name="Wear_Leveling_Count", + attribute_value_type="value"} < 20 + for: 1h + labels: + severity: warning + annotations: + summary: + SATA SSD {{ $labels.device }} on {{ $labels.host }} wear indicator is low ({{ $value }}) + description: "Wear_Leveling_Count normalized value below 20 (100 = new)." diff --git a/hosts/12kingdoms-youko/homelab-services/monitoring/victoriametrics.nix b/hosts/12kingdoms-youko/homelab-services/monitoring/victoriametrics.nix index 2592d7e6..533a6585 100644 --- a/hosts/12kingdoms-youko/homelab-services/monitoring/victoriametrics.nix +++ b/hosts/12kingdoms-youko/homelab-services/monitoring/victoriametrics.nix @@ -19,6 +19,16 @@ let "rakushun" "mitsuha" ]; + + # Physical hosts run smartctl_exporter on :9633 (VM guests have no hardware to + # inspect and don't run it). Keep in sync with the physical machines in + # `vars/networking.nix`. + physicalHosts = [ + "ai" + "shoryu" + "shushou" + "youko" + ]; in { # Since victoriametrics use DynamicUser, the user & group do not exists before the service starts. @@ -251,7 +261,29 @@ in ) [ ] (lib.attrsets.filterAttrs (n: _: !(builtins.elem n offlineHosts)) myvars.networking.hostsAddr) - ); + ) + # --- Drive/SSD SMART health (physical hosts only) --- # + ++ (lib.attrsets.foldlAttrs ( + acc: hostname: addr: + acc + ++ [ + { + job_name = "smartctl-${hostname}"; + scrape_interval = "60s"; + metrics_path = "/metrics"; + static_configs = [ + { + targets = [ "${addr.ipv4}:9633" ]; + labels.type = "smartctl"; + labels.app = "smartctl"; + labels.host = hostname; + labels.env = "homelab"; + labels.cluster = "homelab"; + } + ]; + } + ] + ) [ ] (lib.attrsets.filterAttrs (n: _: builtins.elem n physicalHosts) myvars.networking.hostsAddr)); }; }; } diff --git a/hosts/idols-ai/default.nix b/hosts/idols-ai/default.nix index 3f806bb8..426a108a 100644 --- a/hosts/idols-ai/default.nix +++ b/hosts/idols-ai/default.nix @@ -69,6 +69,12 @@ in RuntimeMaxUse = "256M"; }; + # Monitor SSD/NVMe SMART attributes so a drive going bad is visible before it + # fails, and so there is data behind the relaxed NVMe power-management + # settings in hardware-configuration.nix. smartd autodetects devices and + # reports to the journal (mail notifications stay off without an MTA). + services.smartd.enable = true; + # QEMU/KVM(HostCpuOnly) for local VMs. environment.systemPackages = [ pkgs.qemu_kvm ]; diff --git a/hosts/idols-ai/hardware-configuration.nix b/hosts/idols-ai/hardware-configuration.nix index 9b7cf978..ffcb12f1 100644 --- a/hosts/idols-ai/hardware-configuration.nix +++ b/hosts/idols-ai/hardware-configuration.nix @@ -14,24 +14,25 @@ ]; boot.kernelParams = [ - # === NVMe SSD Timeout / Freeze Fix for Linux === - # https://community.frame.work/t/nvme-timeout-woes/54999 - - "nvme_core.default_ps_max_latency_us=0" - # Explanation: Completely disables NVMe Autonomous Power State Transition (APST) - # Why: Your drive enters deep sleep states during high load. Wake-up latency is too slow (>30 ms), - # causing the kernel to think the command timed out. - # Setting it to 0 = never let the drive sleep → root-cause fix for "freezes during big reads/writes" - - "nvme_core.io_timeout=4294967295" - # Explanation: Increases the kernel's NVMe command timeout to the maximum possible value (~49 days) - # Why: Linux default is only 30 seconds, after which it aborts the request and resets the controller. - # This makes the kernel "patient" so even if the drive is momentarily slow, it won't crash/reset. - - "pcie_aspm=off" - # Explanation: Fully disables PCIe Active State Power Management (link power saving) - # Why: The PCIe link dropping into L1/L1.2 low-power states is the #1 cause of NVMe timeouts on Linux. - # Turning it off keeps the link at full speed at all times → eliminates "Link is Down" + timeout errors. + # === NVMe power management (relaxed 2026-10) === + # + # These began as a hard workaround for NVMe freezes on the KINGBANK KP260 + # (a DRAM-less QLC drive, prone to slow APST wakeups): APST fully disabled, + # the command timeout pushed to the maximum, and PCIe ASPM turned off. + # Kernel logs from 2026-09-19..10-03 (8 boots) show no NVMe + # timeout/reset/AER, so it is relaxed to reclaim idle power and let the + # kernel recover a stuck command instead of hanging on it: + # + # - APST allowed again, capped at a wide 100 ms latency (was: disabled). + # - io_timeout back to the kernel default 30 s (was: ~49 days, which let + # a stuck command wait forever instead of resetting the controller). + # - pcie_aspm removed: PCIe link power saving is unrelated to the drive's + # own APST. + # + # If freezes return under heavy read/write, restore the old workaround: + # nvme_core.default_ps_max_latency_us=0 nvme_core.io_timeout=4294967295 pcie_aspm=off + # See https://community.frame.work/t/nvme-timeout-woes/54999 + "nvme_core.default_ps_max_latency_us=100000" ]; # Use the EFI boot loader. diff --git a/modules/nixos/base/monitoring.nix b/modules/nixos/base/monitoring.nix index cb555734..06e7cdd0 100644 --- a/modules/nixos/base/monitoring.nix +++ b/modules/nixos/base/monitoring.nix @@ -1,3 +1,9 @@ +{ + config, + lib, + myvars, + ... +}: { # enable the node exporter on all nixos hosts # https://github.com/NixOS/nixpkgs/blob/nixos-26.05/nixos/modules/services/monitoring/prometheus/exporters/node.nix @@ -33,4 +39,19 @@ "--collector.filesystem.mount-points-exclude=^(/|/persistent/)(dev|proc|sys|run/credentials/.+|run/user/.+|var/lib/docker/.+|var/lib/containers/.+|var/lib/kubelet/.+|home/ryan/.+)($|/)" ]; }; + + # Drive SMART health (wear, spare, media errors, temperature, bytes written) + # on physical hosts. node_exporter's nvme collector only exposes + # node_nvme_info and namespace capacity, NOT SMART health, so this is the only + # source of SSD lifespan data. `hardwareTools` is off on MicroVM/QEMU guests, + # which have no physical disks to inspect. + services.prometheus.exporters.smartctl = lib.mkIf config.modules.hardwareTools.enable { + enable = true; + # Bind to this host's LAN address rather than 0.0.0.0; the firewall also + # drops 9633 from anything but the monitoring host. Fall back to loopback + # for hosts without a static LAN address (e.g. shoukei). + listenAddress = myvars.networking.hostsAddr.${config.networking.hostName}.ipv4 or "127.0.0.1"; + port = 9633; + maxInterval = "60s"; + }; } diff --git a/modules/nixos/base/networking/firewall.nix b/modules/nixos/base/networking/firewall.nix index 72c6b466..fe405a1c 100644 --- a/modules/nixos/base/networking/firewall.nix +++ b/modules/nixos/base/networking/firewall.nix @@ -29,6 +29,10 @@ in # monitoring host. ip saddr != ${monitoringHost} tcp dport 9835 drop + # smartctl_exporter (drive/SSD SMART health) is scraped only by the + # monitoring host. + ip saddr != ${monitoringHost} tcp dport 9633 drop + # The local container bridge and the tailnet have the same access as the # LAN. Kept after the drop above so neither can reach node_exporter. iifname "podman0" accept