mirror of
https://github.com/ryan4yin/nix-config.git
synced 2026-11-13 12:13:46 +01:00
feat(storage): relax ai NVMe workarounds and add drive SMART monitoring (#359)
* fix(ai): relax NVMe power-management workarounds and enable smartd Logs show no NVMe errors, so allow APST again (capped at 100 ms), restore the default io_timeout, and drop pcie_aspm=off. Add smartd for drive health visibility. * feat(storage): add drive/SSD SMART monitoring on physical hosts node_exporter's nvme collector only exposes node_nvme_info and namespace capacity, not SMART health, so SSD wear/lifespan and drive temperature were not observable anywhere. - enable services.prometheus.exporters.smartctl on physical hosts, gated on modules.hardwareTools (off on QEMU/MicroVM guests), bound to the host's LAN address on port 9633 - firewall: drop 9633 from anything but the monitoring host - youko: scrape smartctl-<host> for the physical hosts - alert rules (storage.yml): SMART health, NVMe wear, available spare, media errors, critical warning, and drive temperature (warn 70 / crit 80 C) - Grafana dashboard "Drive SMART (smartctl)"
This commit is contained in:
ryan4yin | 二花 authored and 
GitHub committed
1 parent
8bee7d93a7
commit
dc3ec8c159
8 files changed
+773
-19
No files matched your search
@@ -4,6 +4,8 @@
|
||||
|
||||
1. https://grafana.com/grafana/dashboards/1860-node-exporter-full/
|
||||
2. https://grafana.com/grafana/dashboards/9578-alertmanager/
|
||||
3. `smartctl.json` — drive/SSD SMART health (wear, spare, media errors, temperature). Custom, built
|
||||
on the `smartctl_exporter` metrics.
|
||||
|
||||
## Kubernetes
|
||||
|
||||
|
||||
@@ -0,0 +1,584 @@
|
||||
{
|
||||
"__inputs": [],
|
||||
"__elements": {},
|
||||
"__requires": [
|
||||
{
|
||||
"type": "grafana",
|
||||
"id": "grafana",
|
||||
"name": "Grafana",
|
||||
"version": "11.2.0"
|
||||
},
|
||||
{
|
||||
"type": "datasource",
|
||||
"id": "prometheus",
|
||||
"name": "Prometheus",
|
||||
"version": "1.0.0"
|
||||
},
|
||||
{
|
||||
"type": "panel",
|
||||
"id": "stat",
|
||||
"name": "Stat",
|
||||
"version": ""
|
||||
},
|
||||
{
|
||||
"type": "panel",
|
||||
"id": "timeseries",
|
||||
"name": "Time series",
|
||||
"version": ""
|
||||
}
|
||||
],
|
||||
"annotations": {
|
||||
"list": [
|
||||
{
|
||||
"builtIn": 1,
|
||||
"datasource": {
|
||||
"type": "grafana",
|
||||
"uid": "grafana"
|
||||
},
|
||||
"enable": true,
|
||||
"hide": true,
|
||||
"iconColor": "rgba(0, 211, 255, 1)",
|
||||
"name": "Annotations & Alerts",
|
||||
"type": "dashboard"
|
||||
}
|
||||
]
|
||||
},
|
||||
"editable": true,
|
||||
"fiscalYearStartMonth": 0,
|
||||
"graphTooltip": 0,
|
||||
"id": null,
|
||||
"links": [],
|
||||
"panels": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"mappings": [
|
||||
{
|
||||
"options": {
|
||||
"0": {
|
||||
"color": "red",
|
||||
"text": "FAILED"
|
||||
},
|
||||
"1": {
|
||||
"color": "green",
|
||||
"text": "OK"
|
||||
}
|
||||
},
|
||||
"type": "value"
|
||||
}
|
||||
],
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "red",
|
||||
"value": null
|
||||
},
|
||||
{
|
||||
"color": "green",
|
||||
"value": 1
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 4,
|
||||
"w": 6,
|
||||
"x": 0,
|
||||
"y": 0
|
||||
},
|
||||
"id": 1,
|
||||
"options": {
|
||||
"colorMode": "background",
|
||||
"graphMode": "none",
|
||||
"justifyMode": "auto",
|
||||
"orientation": "auto",
|
||||
"reduceOptions": {
|
||||
"calcs": ["lastNotNull"],
|
||||
"fields": "",
|
||||
"values": false
|
||||
},
|
||||
"textMode": "auto"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"expr": "smartctl_device_smart_status{host=~\"$host\",device=~\"$device\"}",
|
||||
"instant": true,
|
||||
"legendFormat": "{{host}} {{device}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "SMART health",
|
||||
"type": "stat"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "h"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 4,
|
||||
"w": 6,
|
||||
"x": 6,
|
||||
"y": 0
|
||||
},
|
||||
"id": 2,
|
||||
"options": {
|
||||
"colorMode": "value",
|
||||
"graphMode": "none",
|
||||
"justifyMode": "auto",
|
||||
"orientation": "auto",
|
||||
"reduceOptions": {
|
||||
"calcs": ["lastNotNull"],
|
||||
"fields": "",
|
||||
"values": false
|
||||
},
|
||||
"textMode": "auto"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"expr": "smartctl_device_power_on_seconds{host=~\"$host\",device=~\"$device\"} / 3600",
|
||||
"instant": true,
|
||||
"legendFormat": "{{host}} {{device}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "Power-on hours",
|
||||
"type": "stat"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "thresholds"
|
||||
},
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
},
|
||||
{
|
||||
"color": "red",
|
||||
"value": 1
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 4,
|
||||
"w": 6,
|
||||
"x": 12,
|
||||
"y": 0
|
||||
},
|
||||
"id": 3,
|
||||
"options": {
|
||||
"colorMode": "value",
|
||||
"graphMode": "none",
|
||||
"justifyMode": "auto",
|
||||
"orientation": "auto",
|
||||
"reduceOptions": {
|
||||
"calcs": ["lastNotNull"],
|
||||
"fields": "",
|
||||
"values": false
|
||||
},
|
||||
"textMode": "auto"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"expr": "smartctl_device_media_errors{host=~\"$host\",device=~\"$device\"}",
|
||||
"instant": true,
|
||||
"legendFormat": "{{host}} {{device}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "Media errors",
|
||||
"type": "stat"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "thresholds"
|
||||
},
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
},
|
||||
{
|
||||
"color": "red",
|
||||
"value": 1
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 4,
|
||||
"w": 6,
|
||||
"x": 18,
|
||||
"y": 0
|
||||
},
|
||||
"id": 4,
|
||||
"options": {
|
||||
"colorMode": "value",
|
||||
"graphMode": "none",
|
||||
"justifyMode": "auto",
|
||||
"orientation": "auto",
|
||||
"reduceOptions": {
|
||||
"calcs": ["lastNotNull"],
|
||||
"fields": "",
|
||||
"values": false
|
||||
},
|
||||
"textMode": "auto"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"expr": "smartctl_device_critical_warning{host=~\"$host\",device=~\"$device\"}",
|
||||
"instant": true,
|
||||
"legendFormat": "{{host}} {{device}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "NVMe critical warning",
|
||||
"type": "stat"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {
|
||||
"drawStyle": "line",
|
||||
"fillOpacity": 10,
|
||||
"lineWidth": 2,
|
||||
"showPoints": "never"
|
||||
},
|
||||
"unit": "celsius"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 4
|
||||
},
|
||||
"id": 5,
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "desc"
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"expr": "max by (host, device) (smartctl_device_temperature{host=~\"$host\",device=~\"$device\"})",
|
||||
"legendFormat": "{{host}} {{device}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "Temperature",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {
|
||||
"drawStyle": "line",
|
||||
"fillOpacity": 10,
|
||||
"lineWidth": 2,
|
||||
"showPoints": "never"
|
||||
},
|
||||
"max": 100,
|
||||
"min": 0,
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
},
|
||||
{
|
||||
"color": "yellow",
|
||||
"value": 80
|
||||
},
|
||||
{
|
||||
"color": "red",
|
||||
"value": 90
|
||||
}
|
||||
]
|
||||
},
|
||||
"unit": "percent"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 4
|
||||
},
|
||||
"id": 6,
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "desc"
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"expr": "max by (host, device) (smartctl_device_percentage_used{host=~\"$host\",device=~\"$device\"})",
|
||||
"legendFormat": "{{host}} {{device}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "SSD wear (percentage used)",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {
|
||||
"drawStyle": "line",
|
||||
"fillOpacity": 10,
|
||||
"lineWidth": 2,
|
||||
"showPoints": "never"
|
||||
},
|
||||
"max": 100,
|
||||
"min": 0,
|
||||
"unit": "percent"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 12
|
||||
},
|
||||
"id": 7,
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "desc"
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"expr": "max by (host, device) (smartctl_device_available_spare{host=~\"$host\",device=~\"$device\"})",
|
||||
"legendFormat": "{{host}} {{device}} spare",
|
||||
"refId": "A"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"expr": "max by (host, device) (smartctl_device_available_spare_threshold{host=~\"$host\",device=~\"$device\"})",
|
||||
"legendFormat": "{{host}} {{device}} threshold",
|
||||
"refId": "B"
|
||||
}
|
||||
],
|
||||
"title": "Available spare",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {
|
||||
"drawStyle": "line",
|
||||
"fillOpacity": 10,
|
||||
"lineWidth": 2,
|
||||
"showPoints": "never"
|
||||
},
|
||||
"unit": "Bps"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 12
|
||||
},
|
||||
"id": 8,
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "desc"
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"expr": "rate(smartctl_device_bytes_written{host=~\"$host\",device=~\"$device\"}[5m])",
|
||||
"legendFormat": "{{host}} {{device}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "Bytes written (per second)",
|
||||
"type": "timeseries"
|
||||
}
|
||||
],
|
||||
"refresh": "1m",
|
||||
"schemaVersion": 39,
|
||||
"tags": ["homelab", "smartctl", "storage", "ssd"],
|
||||
"templating": {
|
||||
"list": [
|
||||
{
|
||||
"current": {},
|
||||
"hide": 0,
|
||||
"includeAll": false,
|
||||
"label": "Datasource",
|
||||
"multi": false,
|
||||
"name": "DS_PROMETHEUS",
|
||||
"options": [],
|
||||
"query": "prometheus",
|
||||
"refresh": 1,
|
||||
"regex": "",
|
||||
"skipUrlSync": false,
|
||||
"type": "datasource"
|
||||
},
|
||||
{
|
||||
"allValue": ".*",
|
||||
"current": {},
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"includeAll": true,
|
||||
"label": "Host",
|
||||
"multi": true,
|
||||
"name": "host",
|
||||
"options": [],
|
||||
"query": "label_values(smartctl_device, host)",
|
||||
"refresh": 1,
|
||||
"regex": "",
|
||||
"skipUrlSync": false,
|
||||
"sort": 1,
|
||||
"type": "query"
|
||||
},
|
||||
{
|
||||
"allValue": ".*",
|
||||
"current": {},
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"includeAll": true,
|
||||
"label": "Device",
|
||||
"multi": true,
|
||||
"name": "device",
|
||||
"options": [],
|
||||
"query": "label_values(smartctl_device{host=~\"$host\"}, device)",
|
||||
"refresh": 1,
|
||||
"regex": "",
|
||||
"skipUrlSync": false,
|
||||
"sort": 1,
|
||||
"type": "query"
|
||||
}
|
||||
]
|
||||
},
|
||||
"time": {
|
||||
"from": "now-6h",
|
||||
"to": "now"
|
||||
},
|
||||
"timepicker": {},
|
||||
"timezone": "browser",
|
||||
"title": "Drive SMART (smartctl)",
|
||||
"uid": "smartctl-homelab",
|
||||
"version": 1
|
||||
}
|
||||
@@ -0,0 +1,104 @@
|
||||
groups:
|
||||
- name: storage.rules
|
||||
rules:
|
||||
# SMART "overall-health self-assessment" (ATA/SAS) and NVMe health status.
|
||||
# 0 means the drive failed its own health check.
|
||||
- alert: DriveHealthFailed
|
||||
expr: smartctl_device_smart_status == 0
|
||||
for: 10m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: Drive {{ $labels.device }} on {{ $labels.host }} reports failing SMART health
|
||||
description:
|
||||
"{{ $labels.model_name }} (serial {{ $labels.serial_number }}) failed its SMART
|
||||
overall-health self-assessment."
|
||||
|
||||
# NVMe wear: percentage of the rated write endurance already consumed.
|
||||
- alert: SSDWearHigh
|
||||
expr: smartctl_device_percentage_used > 80
|
||||
for: 1h
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: SSD {{ $labels.device }} on {{ $labels.host }} is {{ $value }}% worn
|
||||
description:
|
||||
"{{ $labels.model_name }} (serial {{ $labels.serial_number }}) has consumed {{ $value
|
||||
}}% of its rated write endurance."
|
||||
|
||||
- alert: SSDWearCritical
|
||||
expr: smartctl_device_percentage_used > 90
|
||||
for: 10m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: SSD {{ $labels.device }} on {{ $labels.host }} is {{ $value }}% worn
|
||||
description:
|
||||
"{{ $labels.model_name }} is nearly out of write endurance; plan a replacement."
|
||||
|
||||
# NVMe available spare fell below the drive's own threshold.
|
||||
- alert: SSDSpareLow
|
||||
expr: smartctl_device_available_spare < smartctl_device_available_spare_threshold
|
||||
for: 10m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: SSD {{ $labels.device }} on {{ $labels.host }} spare blocks below threshold
|
||||
description:
|
||||
"Available spare {{ $value }}% is below the drive threshold; the controller may
|
||||
transition to read-only."
|
||||
|
||||
# NVMe critical warning bitfield (spare low, temperature, reliability,
|
||||
# read-only, volatile-memory backup).
|
||||
- alert: SSDControllerCriticalWarning
|
||||
expr: smartctl_device_critical_warning > 0
|
||||
for: 10m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: SSD {{ $labels.device }} on {{ $labels.host }} has a controller critical warning
|
||||
description: "smartctl_device_critical_warning = {{ $value }} (non-zero bitfield)."
|
||||
|
||||
# Unrecovered data-integrity errors (uncorrectable ECC, CRC, LBA tag).
|
||||
- alert: DriveMediaErrors
|
||||
expr: smartctl_device_media_errors > 0
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: Drive {{ $labels.device }} on {{ $labels.host }} has media errors
|
||||
description: "{{ $value }} unrecovered data-integrity errors recorded."
|
||||
|
||||
# Drive temperature (SSD/HDD). Consumer M.2 NVMe throttles around 70-80 C.
|
||||
- alert: DriveTemperatureHigh
|
||||
expr: max by (host, device) (smartctl_device_temperature) > 70
|
||||
for: 15m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: Drive {{ $labels.device }} on {{ $labels.host }} is hot ({{ $value }} C)
|
||||
description: "Drive temperature above 70 C; NVMe throttles around 70-80 C."
|
||||
|
||||
- alert: DriveTemperatureCritical
|
||||
expr: max by (host, device) (smartctl_device_temperature) > 80
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary:
|
||||
Drive {{ $labels.device }} on {{ $labels.host }} is critically hot ({{ $value }} C)
|
||||
description: "Drive temperature above 80 C; risk of throttling or failure."
|
||||
|
||||
# SATA/SAS SSD wear via the ATA Wear_Leveling_Count attribute
|
||||
# (normalized value: 100 = new, lower = more worn).
|
||||
- alert: AtaSSDWearHigh
|
||||
expr:
|
||||
smartctl_device_attribute{attribute_name="Wear_Leveling_Count",
|
||||
attribute_value_type="value"} < 20
|
||||
for: 1h
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary:
|
||||
SATA SSD {{ $labels.device }} on {{ $labels.host }} wear indicator is low ({{ $value }})
|
||||
description: "Wear_Leveling_Count normalized value below 20 (100 = new)."
|
||||
@@ -19,6 +19,16 @@ let
|
||||
"rakushun"
|
||||
"mitsuha"
|
||||
];
|
||||
|
||||
# Physical hosts run smartctl_exporter on :9633 (VM guests have no hardware to
|
||||
# inspect and don't run it). Keep in sync with the physical machines in
|
||||
# `vars/networking.nix`.
|
||||
physicalHosts = [
|
||||
"ai"
|
||||
"shoryu"
|
||||
"shushou"
|
||||
"youko"
|
||||
];
|
||||
in
|
||||
{
|
||||
# Since victoriametrics use DynamicUser, the user & group do not exists before the service starts.
|
||||
@@ -251,7 +261,29 @@ in
|
||||
)
|
||||
[ ]
|
||||
(lib.attrsets.filterAttrs (n: _: !(builtins.elem n offlineHosts)) myvars.networking.hostsAddr)
|
||||
);
|
||||
)
|
||||
# --- Drive/SSD SMART health (physical hosts only) --- #
|
||||
++ (lib.attrsets.foldlAttrs (
|
||||
acc: hostname: addr:
|
||||
acc
|
||||
++ [
|
||||
{
|
||||
job_name = "smartctl-${hostname}";
|
||||
scrape_interval = "60s";
|
||||
metrics_path = "/metrics";
|
||||
static_configs = [
|
||||
{
|
||||
targets = [ "${addr.ipv4}:9633" ];
|
||||
labels.type = "smartctl";
|
||||
labels.app = "smartctl";
|
||||
labels.host = hostname;
|
||||
labels.env = "homelab";
|
||||
labels.cluster = "homelab";
|
||||
}
|
||||
];
|
||||
}
|
||||
]
|
||||
) [ ] (lib.attrsets.filterAttrs (n: _: builtins.elem n physicalHosts) myvars.networking.hostsAddr));
|
||||
};
|
||||
};
|
||||
}
|
||||
@@ -69,6 +69,12 @@ in
|
||||
RuntimeMaxUse = "256M";
|
||||
};
|
||||
|
||||
# Monitor SSD/NVMe SMART attributes so a drive going bad is visible before it
|
||||
# fails, and so there is data behind the relaxed NVMe power-management
|
||||
# settings in hardware-configuration.nix. smartd autodetects devices and
|
||||
# reports to the journal (mail notifications stay off without an MTA).
|
||||
services.smartd.enable = true;
|
||||
|
||||
# QEMU/KVM(HostCpuOnly) for local VMs.
|
||||
environment.systemPackages = [ pkgs.qemu_kvm ];
|
||||
|
||||
|
||||
@@ -14,24 +14,25 @@
|
||||
];
|
||||
|
||||
boot.kernelParams = [
|
||||
# === NVMe SSD Timeout / Freeze Fix for Linux ===
|
||||
# https://community.frame.work/t/nvme-timeout-woes/54999
|
||||
|
||||
"nvme_core.default_ps_max_latency_us=0"
|
||||
# Explanation: Completely disables NVMe Autonomous Power State Transition (APST)
|
||||
# Why: Your drive enters deep sleep states during high load. Wake-up latency is too slow (>30 ms),
|
||||
# causing the kernel to think the command timed out.
|
||||
# Setting it to 0 = never let the drive sleep → root-cause fix for "freezes during big reads/writes"
|
||||
|
||||
"nvme_core.io_timeout=4294967295"
|
||||
# Explanation: Increases the kernel's NVMe command timeout to the maximum possible value (~49 days)
|
||||
# Why: Linux default is only 30 seconds, after which it aborts the request and resets the controller.
|
||||
# This makes the kernel "patient" so even if the drive is momentarily slow, it won't crash/reset.
|
||||
|
||||
"pcie_aspm=off"
|
||||
# Explanation: Fully disables PCIe Active State Power Management (link power saving)
|
||||
# Why: The PCIe link dropping into L1/L1.2 low-power states is the #1 cause of NVMe timeouts on Linux.
|
||||
# Turning it off keeps the link at full speed at all times → eliminates "Link is Down" + timeout errors.
|
||||
# === NVMe power management (relaxed 2026-10) ===
|
||||
#
|
||||
# These began as a hard workaround for NVMe freezes on the KINGBANK KP260
|
||||
# (a DRAM-less QLC drive, prone to slow APST wakeups): APST fully disabled,
|
||||
# the command timeout pushed to the maximum, and PCIe ASPM turned off.
|
||||
# Kernel logs from 2026-09-19..10-03 (8 boots) show no NVMe
|
||||
# timeout/reset/AER, so it is relaxed to reclaim idle power and let the
|
||||
# kernel recover a stuck command instead of hanging on it:
|
||||
#
|
||||
# - APST allowed again, capped at a wide 100 ms latency (was: disabled).
|
||||
# - io_timeout back to the kernel default 30 s (was: ~49 days, which let
|
||||
# a stuck command wait forever instead of resetting the controller).
|
||||
# - pcie_aspm removed: PCIe link power saving is unrelated to the drive's
|
||||
# own APST.
|
||||
#
|
||||
# If freezes return under heavy read/write, restore the old workaround:
|
||||
# nvme_core.default_ps_max_latency_us=0 nvme_core.io_timeout=4294967295 pcie_aspm=off
|
||||
# See https://community.frame.work/t/nvme-timeout-woes/54999
|
||||
"nvme_core.default_ps_max_latency_us=100000"
|
||||
];
|
||||
|
||||
# Use the EFI boot loader.
|
||||
|
||||
@@ -1,3 +1,9 @@
|
||||
{
|
||||
config,
|
||||
lib,
|
||||
myvars,
|
||||
...
|
||||
}:
|
||||
{
|
||||
# enable the node exporter on all nixos hosts
|
||||
# https://github.com/NixOS/nixpkgs/blob/nixos-26.05/nixos/modules/services/monitoring/prometheus/exporters/node.nix
|
||||
@@ -33,4 +39,19 @@
|
||||
"--collector.filesystem.mount-points-exclude=^(/|/persistent/)(dev|proc|sys|run/credentials/.+|run/user/.+|var/lib/docker/.+|var/lib/containers/.+|var/lib/kubelet/.+|home/ryan/.+)($|/)"
|
||||
];
|
||||
};
|
||||
|
||||
# Drive SMART health (wear, spare, media errors, temperature, bytes written)
|
||||
# on physical hosts. node_exporter's nvme collector only exposes
|
||||
# node_nvme_info and namespace capacity, NOT SMART health, so this is the only
|
||||
# source of SSD lifespan data. `hardwareTools` is off on MicroVM/QEMU guests,
|
||||
# which have no physical disks to inspect.
|
||||
services.prometheus.exporters.smartctl = lib.mkIf config.modules.hardwareTools.enable {
|
||||
enable = true;
|
||||
# Bind to this host's LAN address rather than 0.0.0.0; the firewall also
|
||||
# drops 9633 from anything but the monitoring host. Fall back to loopback
|
||||
# for hosts without a static LAN address (e.g. shoukei).
|
||||
listenAddress = myvars.networking.hostsAddr.${config.networking.hostName}.ipv4 or "127.0.0.1";
|
||||
port = 9633;
|
||||
maxInterval = "60s";
|
||||
};
|
||||
}
|
||||
@@ -29,6 +29,10 @@ in
|
||||
# monitoring host.
|
||||
ip saddr != ${monitoringHost} tcp dport 9835 drop
|
||||
|
||||
# smartctl_exporter (drive/SSD SMART health) is scraped only by the
|
||||
# monitoring host.
|
||||
ip saddr != ${monitoringHost} tcp dport 9633 drop
|
||||
|
||||
# The local container bridge and the tailnet have the same access as the
|
||||
# LAN. Kept after the drop above so neither can reach node_exporter.
|
||||
iifname "podman0" accept
|
||||
|
||||
Reference in new issue
Block a user