From fabccbafda4b7d705982d8ef1419ea1098613f74 Mon Sep 17 00:00:00 2001 From: Ryan Yin Date: Wed, 7 Oct 2026 13:21:43 +0800 Subject: [PATCH] fix(monitoring): ignore openapi discovery in apiserver latency alert KubernetesApiServerLatency fired continuously on the p99 of apiserver_request_duration_seconds_bucket. Every firing series was verb="GET", subresource="openapi/v3" (kubelet, kube-controller-manager, kube-scheduler, apiserver) at ~8s, while real verbs (POST/LIST/DELETE) stay around 0.2s. openapi/v3 is a streaming discovery endpoint, like WATCH, so exclude subresource=~"^openapi/" from the rule. --- .../monitoring/alert_rules/kube-state-metrics-exporter.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hosts/12kingdoms-youko/homelab-services/monitoring/alert_rules/kube-state-metrics-exporter.yml b/hosts/12kingdoms-youko/homelab-services/monitoring/alert_rules/kube-state-metrics-exporter.yml index 9a839f15..872e49f7 100644 --- a/hosts/12kingdoms-youko/homelab-services/monitoring/alert_rules/kube-state-metrics-exporter.yml +++ b/hosts/12kingdoms-youko/homelab-services/monitoring/alert_rules/kube-state-metrics-exporter.yml @@ -422,7 +422,7 @@ groups: - alert: KubernetesApiServerLatency expr: 'histogram_quantile(0.99, - sum(rate(apiserver_request_duration_seconds_bucket{subresource!="log",verb!~"^(?:CONNECT|WATCHLIST|WATCH|PROXY)$"} + sum(rate(apiserver_request_duration_seconds_bucket{subresource!="log",subresource!~"^openapi/",verb!~"^(?:CONNECT|WATCHLIST|WATCH|PROXY)$"} [10m])) WITHOUT (instance, resource)) > 1' for: 2m labels: