From 3d3be75718494221b31aedd2f9c3fa565d301f4d Mon Sep 17 00:00:00 2001 From: Ryan Yin Date: Sat, 17 Oct 2026 22:59:00 +0800 Subject: [PATCH] fix(monitoring): use per-component temperature thresholds CPU warns at 90 C, GPU at 95 C and other components at 75 C; per-core probes are excluded so a full-core compile no longer raises 26 alerts. Assisted-by: DeepSeek Harness (deepseek-flash) --- .../monitoring/alert_rules/node-exporter.yml | 36 +++++++++++++++++-- 1 file changed, 33 insertions(+), 3 deletions(-) diff --git a/hosts/12kingdoms-youko/homelab-services/monitoring/alert_rules/node-exporter.yml b/hosts/12kingdoms-youko/homelab-services/monitoring/alert_rules/node-exporter.yml index 51a4e11f..6160df4e 100644 --- a/hosts/12kingdoms-youko/homelab-services/monitoring/alert_rules/node-exporter.yml +++ b/hosts/12kingdoms-youko/homelab-services/monitoring/alert_rules/node-exporter.yml @@ -306,18 +306,48 @@ groups: "btrbk-btrbk.service is in a failed state; btrfs snapshots are not being created.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" + - alert: HostCpuTooHot + expr: + '((node_hwmon_temp_celsius * ignoring(label) group_left(instance, job, node, sensor) + node_hwmon_sensor_label{label=~"(?i)^(tctl|tccd|tdie|package.*|peci.*)$"} > 90)) * + on(instance) group_left (nodename) node_uname_info{nodename=~".+"}' + for: 5m + labels: + severity: warning + annotations: + summary: Host CPU too hot (instance {{ $labels.instance }}) + description: + "CPU package/die temperature is above 90 C\n VALUE = {{ $value }}\n LABELS = {{ + $labels }}" + + - alert: HostGpuTooHot + expr: + '((node_hwmon_temp_celsius * ignoring(label) group_left(instance, job, node, sensor) + node_hwmon_sensor_label{label=~"(?i)^(edge|junction|mem)$"} > 95)) * on(instance) + group_left (nodename) node_uname_info{nodename=~".+"}' + for: 5m + labels: + severity: warning + annotations: + summary: Host GPU too hot (instance {{ $labels.instance }}) + description: + "GPU edge/junction/memory temperature is above 95 C\n VALUE = {{ $value }}\n LABELS = + {{ $labels }}" + - alert: HostPhysicalComponentTooHot expr: '((node_hwmon_temp_celsius * ignoring(label) group_left(instance, job, node, sensor) - node_hwmon_sensor_label{label!~"(?i)^(tctl|tccd|tdie|auxtin[0-9]*|cputin)$"} > 75)) * - on(instance) group_left (nodename) node_uname_info{nodename=~".+"}' + node_hwmon_sensor_label{label!~"(?i)^(tctl|tccd|tdie|aux.*|cputin|package.*|core + .*|peci.*|edge|junction|mem)$"} > 75)) * on(instance) group_left (nodename) + node_uname_info{nodename=~".+"}' for: 5m labels: severity: warning annotations: summary: Host physical component too hot (instance {{ $labels.instance }}) description: - "Physical hardware component too hot\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" + "Non-CPU/GPU hardware component too hot\n VALUE = {{ $value }}\n LABELS = {{ $labels + }}" - alert: HostNodeOvertemperatureAlarm expr: