fix(monitoring): bound telegram message length and mute meta alerts

- Trim the telegram template: drop the full label dump, render at
  most 5 alerts per group, and note how many were omitted, so
  messages stay within Telegram's 4096-char limit.
- Route severity=none meta alerts (Watchdog, InfoInhibitor) to a new
  null receiver so they no longer notify.

Signed-off-by: Ryan Yin <xiaoyin_c@qq.com>
This commit is contained in:
Ryan Yin
2026-07-24 20:57:54 +08:00
parent 149d8bc137
commit 35d41a4391
+30 -11
View File
@@ -51,6 +51,12 @@
route = {
receiver = "telegram";
routes = [
{
# Watchdog & InfoInhibitor are meta alerts that should never notify,
# route them to a null receiver.
receiver = "null";
matchers = [ ''severity = "none"'' ];
}
{
receiver = "telegram";
# group alerts by labels
@@ -90,6 +96,10 @@
];
};
receivers = [
{
# Discards all notifications (for meta alerts that should never notify).
name = "null";
}
# {
# name = "email";
# email_configs = [
@@ -114,25 +124,34 @@
# https://core.telegram.org/bots/api#formatting-options
parse_mode = "HTML";
# Message template
# Telegram limits a message to 4096 chars, so only render the first 5 alerts
# in a group and keep the fields minimal (no full label dump).
message = ''
{{- if eq .Status "firing" }}
🟡 <b></b> {{ .CommonLabels.alertname }} [{{ index .CommonLabels "severity" | title }}]
🟡 <b></b> {{ .CommonLabels.alertname }} [{{ index .CommonLabels "severity" | title }}] ( {{ len .Alerts }} )
{{- else }}
🟢 <b></b> {{ .CommonLabels.alertname }} [{{ index .CommonLabels "severity" | title }}]
{{- end }}
{{- range .Alerts }}
{{- range $i, $a := .Alerts }}
{{- if lt $i 5 }}
📊 <b>:</b>
<b></b>: {{ .Labels.alertgroup }}
<b></b>: {{ if eq .Labels.severity "critical" }}🔴{{ else }}🟡 {{ end }} {{ .Labels.severity | title }}
<b></b>: <a href="{{ .GeneratorURL }}">Grafana Explore</a>
<b></b>: {{ with .Annotations.value }}{{ . }}{{ else }}N/A{{ end }}
<b>Env</b>: {{ with .Labels.env }}{{ . }}{{ else }}N/A{{ end }}
<b>Cluster</b>: {{ with .Labels.cluster }}{{ . }}{{ else }}N/A{{ end }}
<b>Namespace</b>: {{ with .Labels.namespace }}{{ . }}{{ else }}N/A{{ end }}
<b></b>: {{ range .Labels.SortedPairs }}{{ .Name }}={{ .Value }},{{ end }}
<b></b>: {{ .StartsAt.Format "2006-01-02 15:04:05" }}
<b></b>: {{ $a.Labels.alertgroup }}
<b></b>: {{ if eq $a.Labels.severity "critical" }}🔴{{ else }}🟡 {{ end }} {{ $a.Labels.severity | title }}
<b></b>: {{ with $a.Labels.instance }}{{ . }}{{ else }}N/A{{ end }}
<b>Pod</b>: {{ with $a.Labels.pod }}{{ . }}{{ else }}N/A{{ end }}
<b></b>: <a href="{{ $a.GeneratorURL }}">Grafana Explore</a>
<b></b>: {{ with $a.Annotations.value }}{{ . }}{{ else }}N/A{{ end }}
<b>Env</b>: {{ with $a.Labels.env }}{{ . }}{{ else }}N/A{{ end }}
<b>Cluster</b>: {{ with $a.Labels.cluster }}{{ . }}{{ else }}N/A{{ end }}
<b>Namespace</b>: {{ with $a.Labels.namespace }}{{ . }}{{ else }}N/A{{ end }}
<b></b>: {{ $a.StartsAt.Format "2006-01-02 15:04:05" }}
{{- end }}
{{- end }}
{{- if gt (len .Alerts) 5 }}
{{ len (slice .Alerts 5) }} Alertmanager
{{- end }}
'';
}