From 35d41a439171aa95fd00d518e6b3445dbf37aaf9 Mon Sep 17 00:00:00 2001
From: Ryan Yin <xiaoyin_c@qq.com>
Date: Fri, 24 Jul 2026 20:57:54 +0800
Subject: [PATCH] fix(monitoring): bound telegram message length and mute meta
 alerts

- Trim the telegram template: drop the full label dump, render at
  most 5 alerts per group, and note how many were omitted, so
  messages stay within Telegram's 4096-char limit.
- Route severity=none meta alerts (Watchdog, InfoInhibitor) to a new
  null receiver so they no longer notify.

Signed-off-by: Ryan Yin <xiaoyin_c@qq.com>
---
 hosts/idols-aquamarine/monitoring/alert.nix | 41 +++++++++++++++------
 1 file changed, 30 insertions(+), 11 deletions(-)

diff --git a/hosts/idols-aquamarine/monitoring/alert.nix b/hosts/idols-aquamarine/monitoring/alert.nix
index 24ad1d24..c046c158 100644
--- a/hosts/idols-aquamarine/monitoring/alert.nix
+++ b/hosts/idols-aquamarine/monitoring/alert.nix
@@ -51,6 +51,12 @@
       route = {
         receiver = "telegram";
         routes = [
+          {
+            # Watchdog & InfoInhibitor are meta alerts that should never notify,
+            # route them to a null receiver.
+            receiver = "null";
+            matchers = [ ''severity = "none"'' ];
+          }
           {
             receiver = "telegram";
             # group alerts by labels
@@ -90,6 +96,10 @@
         ];
       };
       receivers = [
+        {
+          # Discards all notifications (for meta alerts that should never notify).
+          name = "null";
+        }
         # {
         #   name = "email";
         #   email_configs = [
@@ -114,25 +124,34 @@
               # https://core.telegram.org/bots/api#formatting-options
               parse_mode = "HTML";
               # Message template
+              # Telegram limits a message to 4096 chars, so only render the first 5 alerts
+              # in a group and keep the fields minimal (no full label dump).
               message = ''
                 {{- if eq .Status "firing" }}
-                🟡 <b>告警触发</b>  {{ .CommonLabels.alertname }} [{{ index .CommonLabels "severity" | title }}]
+                🟡 <b>告警触发</b>  {{ .CommonLabels.alertname }} [{{ index .CommonLabels "severity" | title }}] (共 {{ len .Alerts }} 条)
                 {{- else }}
                 🟢 <b>告警恢复</b>  {{ .CommonLabels.alertname }} [{{ index .CommonLabels "severity" | title }}]
                 {{- end }}
 
-                {{- range .Alerts }}
+                {{- range $i, $a := .Alerts }}
+                {{- if lt $i 5 }}
 
                 📊 <b>详情:</b>
-                • <b>告警组</b>: {{ .Labels.alertgroup }}
-                • <b>等级</b>: {{ if eq .Labels.severity "critical" }}🔴{{ else }}🟡 {{ end }} {{ .Labels.severity | title }}
-                • <b>查询</b>: <a href="{{ .GeneratorURL }}">Grafana Explore</a>
-                • <b>触发值</b>: {{ with .Annotations.value }}{{ . }}{{ else }}N/A{{ end }}
-                • <b>Env</b>: {{ with .Labels.env }}{{ . }}{{ else }}N/A{{ end }}
-                • <b>Cluster</b>: {{ with .Labels.cluster }}{{ . }}{{ else }}N/A{{ end }}
-                • <b>Namespace</b>: {{ with .Labels.namespace }}{{ . }}{{ else }}N/A{{ end }}
-                • <b>标签</b>: {{ range .Labels.SortedPairs }}{{ .Name }}={{ .Value }},{{ end }}
-                • <b>触发时间</b>: {{ .StartsAt.Format "2006-01-02 15:04:05" }}
+                • <b>告警组</b>: {{ $a.Labels.alertgroup }}
+                • <b>等级</b>: {{ if eq $a.Labels.severity "critical" }}🔴{{ else }}🟡 {{ end }} {{ $a.Labels.severity | title }}
+                • <b>实例</b>: {{ with $a.Labels.instance }}{{ . }}{{ else }}N/A{{ end }}
+                • <b>Pod</b>: {{ with $a.Labels.pod }}{{ . }}{{ else }}N/A{{ end }}
+                • <b>查询</b>: <a href="{{ $a.GeneratorURL }}">Grafana Explore</a>
+                • <b>触发值</b>: {{ with $a.Annotations.value }}{{ . }}{{ else }}N/A{{ end }}
+                • <b>Env</b>: {{ with $a.Labels.env }}{{ . }}{{ else }}N/A{{ end }}
+                • <b>Cluster</b>: {{ with $a.Labels.cluster }}{{ . }}{{ else }}N/A{{ end }}
+                • <b>Namespace</b>: {{ with $a.Labels.namespace }}{{ . }}{{ else }}N/A{{ end }}
+                • <b>触发时间</b>: {{ $a.StartsAt.Format "2006-01-02 15:04:05" }}
+                {{- end }}
+                {{- end }}
+                {{- if gt (len .Alerts) 5 }}
+
+                ⚠️ 另有 {{ len (slice .Alerts 5) }} 条同组告警未列出，请查看 Alertmanager。
                 {{- end }}
               '';
             }