diff --git a/hosts/idols-aquamarine/monitoring/alert.nix b/hosts/idols-aquamarine/monitoring/alert.nix
index 24ad1d24..c046c158 100644
--- a/hosts/idols-aquamarine/monitoring/alert.nix
+++ b/hosts/idols-aquamarine/monitoring/alert.nix
@@ -51,6 +51,12 @@
route = {
receiver = "telegram";
routes = [
+ {
+ # Watchdog & InfoInhibitor are meta alerts that should never notify,
+ # route them to a null receiver.
+ receiver = "null";
+ matchers = [ ''severity = "none"'' ];
+ }
{
receiver = "telegram";
# group alerts by labels
@@ -90,6 +96,10 @@
];
};
receivers = [
+ {
+ # Discards all notifications (for meta alerts that should never notify).
+ name = "null";
+ }
# {
# name = "email";
# email_configs = [
@@ -114,25 +124,34 @@
# https://core.telegram.org/bots/api#formatting-options
parse_mode = "HTML";
# Message template
+ # Telegram limits a message to 4096 chars, so only render the first 5 alerts
+ # in a group and keep the fields minimal (no full label dump).
message = ''
{{- if eq .Status "firing" }}
- 🟡 告警触发 {{ .CommonLabels.alertname }} [{{ index .CommonLabels "severity" | title }}]
+ 🟡 告警触发 {{ .CommonLabels.alertname }} [{{ index .CommonLabels "severity" | title }}] (共 {{ len .Alerts }} 条)
{{- else }}
🟢 告警恢复 {{ .CommonLabels.alertname }} [{{ index .CommonLabels "severity" | title }}]
{{- end }}
- {{- range .Alerts }}
+ {{- range $i, $a := .Alerts }}
+ {{- if lt $i 5 }}
📊 详情:
- • 告警组: {{ .Labels.alertgroup }}
- • 等级: {{ if eq .Labels.severity "critical" }}🔴{{ else }}🟡 {{ end }} {{ .Labels.severity | title }}
- • 查询: Grafana Explore
- • 触发值: {{ with .Annotations.value }}{{ . }}{{ else }}N/A{{ end }}
- • Env: {{ with .Labels.env }}{{ . }}{{ else }}N/A{{ end }}
- • Cluster: {{ with .Labels.cluster }}{{ . }}{{ else }}N/A{{ end }}
- • Namespace: {{ with .Labels.namespace }}{{ . }}{{ else }}N/A{{ end }}
- • 标签: {{ range .Labels.SortedPairs }}{{ .Name }}={{ .Value }},{{ end }}
- • 触发时间: {{ .StartsAt.Format "2006-01-02 15:04:05" }}
+ • 告警组: {{ $a.Labels.alertgroup }}
+ • 等级: {{ if eq $a.Labels.severity "critical" }}🔴{{ else }}🟡 {{ end }} {{ $a.Labels.severity | title }}
+ • 实例: {{ with $a.Labels.instance }}{{ . }}{{ else }}N/A{{ end }}
+ • Pod: {{ with $a.Labels.pod }}{{ . }}{{ else }}N/A{{ end }}
+ • 查询: Grafana Explore
+ • 触发值: {{ with $a.Annotations.value }}{{ . }}{{ else }}N/A{{ end }}
+ • Env: {{ with $a.Labels.env }}{{ . }}{{ else }}N/A{{ end }}
+ • Cluster: {{ with $a.Labels.cluster }}{{ . }}{{ else }}N/A{{ end }}
+ • Namespace: {{ with $a.Labels.namespace }}{{ . }}{{ else }}N/A{{ end }}
+ • 触发时间: {{ $a.StartsAt.Format "2006-01-02 15:04:05" }}
+ {{- end }}
+ {{- end }}
+ {{- if gt (len .Alerts) 5 }}
+
+ ⚠️ 另有 {{ len (slice .Alerts 5) }} 条同组告警未列出,请查看 Alertmanager。
{{- end }}
'';
}