feat: update alert rules

2026-07-17 01:51:15 +02:00 · 2025-09-11 13:48:01 +08:00
parent e902a9bdb1
commit 7f112010c2
8 changed files with 546 additions and 130 deletions
@@ -0,0 +1,52 @@
+groups:
+  - name: Loki Exporter
+
+    rules:
+      - alert: LokiProcessTooManyRestarts
+        expr: 'changes(process_start_time_seconds{job=~".*loki.*"}[15m]) > 2'
+        for: 0m
+        labels:
+          severity: warning
+        annotations:
+          summary: Loki process too many restarts (instance {{ $labels.instance }})
+          description:
+            "A loki process had too many restarts (target {{ $labels.instance }})\n  VALUE = {{
+            $value }}\n  LABELS = {{ $labels }}"
+
+      - alert: LokiRequestErrors
+        expr:
+          '100 * sum(rate(loki_request_duration_seconds_count{status_code=~"5.."}[1m])) by
+          (namespace, job, route) / sum(rate(loki_request_duration_seconds_count[1m])) by
+          (namespace, job, route) > 10'
+        for: 15m
+        labels:
+          severity: critical
+        annotations:
+          summary: Loki request errors (instance {{ $labels.instance }})
+          description:
+            "The {{ $labels.job }} and {{ $labels.route }} are experiencing errors\n  VALUE = {{
+            $value }}\n  LABELS = {{ $labels }}"
+
+      - alert: LokiRequestPanic
+        expr: "sum(increase(loki_panic_total[10m])) by (namespace, job) > 0"
+        for: 5m
+        labels:
+          severity: critical
+        annotations:
+          summary: Loki request panic (instance {{ $labels.instance }})
+          description:
+            "The {{ $labels.job }} is experiencing {{ printf \"%.2f\" $value }}% increase of
+            panics\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+
+      - alert: LokiRequestLatency
+        expr:
+          '(histogram_quantile(0.99,
+          sum(rate(loki_request_duration_seconds_bucket{route!~"(?i).*tail.*"}[5m])) by (le)))  > 1'
+        for: 5m
+        labels:
+          severity: critical
+        annotations:
+          summary: Loki request latency (instance {{ $labels.instance }})
+          description:
+            "The {{ $labels.job }} {{ $labels.route }} is experiencing {{ printf \"%.2f\" $value }}s
+            99th percentile latency\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"