diff --git a/flake.lock b/flake.lock index c85af856..08f5775f 100644 --- a/flake.lock +++ b/flake.lock @@ -974,11 +974,11 @@ }, "nixpkgs_2": { "locked": { - "lastModified": 1790822859, - "narHash": "sha256-69xHQhAeMAD2wDXO7T2pcOZIF9Sga2W+JkmY2a11Ops=", - "owner": "nixos", + "lastModified": 1791000213, + "narHash": "sha256-qsAQkushLI1nYI/Uz/OAufjcvg50bZBGBq25pe/jcWk=", + "owner": "NixOS", "repo": "nixpkgs", - "rev": "c59305bab2065cfecc4944690d9eedbb56f3a9fa", + "rev": "f71476e8faba8240824d77877d2da77cc42e6d00", "type": "github" }, "original": { diff --git a/hosts/12kingdoms-youko/homelab-services/monitoring/alert_rules/kube-state-metrics-exporter.yml b/hosts/12kingdoms-youko/homelab-services/monitoring/alert_rules/kube-state-metrics-exporter.yml index 872e49f7..d8fc6920 100644 --- a/hosts/12kingdoms-youko/homelab-services/monitoring/alert_rules/kube-state-metrics-exporter.yml +++ b/hosts/12kingdoms-youko/homelab-services/monitoring/alert_rules/kube-state-metrics-exporter.yml @@ -48,8 +48,8 @@ groups: - alert: KubernetesNodeOutOfPodCapacity expr: - 'sum by (node) ((kube_pod_status_phase{phase="Running"} == 1) + on(uid) group_left(node) - (0 * kube_pod_info{pod_template_hash=""})) / sum by (node) + 'sum by (node) ((kube_pod_status_phase{phase="Running"} == 1) + on(namespace, pod) + group_left(node) (0 * kube_pod_info{pod_template_hash=""})) / sum by (node) (kube_node_status_allocatable{resource="pods"}) * 100 > 90' for: 2m labels: diff --git a/hosts/12kingdoms-youko/homelab-services/monitoring/alert_rules/node-exporter.yml b/hosts/12kingdoms-youko/homelab-services/monitoring/alert_rules/node-exporter.yml index 44f543a8..5f91ae88 100644 --- a/hosts/12kingdoms-youko/homelab-services/monitoring/alert_rules/node-exporter.yml +++ b/hosts/12kingdoms-youko/homelab-services/monitoring/alert_rules/node-exporter.yml @@ -166,9 +166,10 @@ groups: - alert: HostUnusualDiskReadLatency expr: - '(rate(node_disk_read_time_seconds_total[1m]) / rate(node_disk_reads_completed_total[1m]) - > 0.1 and rate(node_disk_reads_completed_total[1m]) > 0) * on(instance) group_left - (nodename) node_uname_info{nodename=~".+"}' + '((rate(node_disk_read_time_seconds_total[1m]) / rate(node_disk_reads_completed_total[1m]) + > 0.1) and rate(node_disk_reads_completed_total[1m]) > 5) * on(instance, device) + group_left node_disk_info{rotational="0", device!~"^(dm-|loop|ram|zram|sr|fd).*"} * + on(instance) group_left (nodename) node_uname_info{nodename=~".+"}' for: 2m labels: severity: warning @@ -180,10 +181,11 @@ groups: - alert: HostUnusualDiskWriteLatency expr: - '(rate(node_disk_write_time_seconds_total[1m]) / - rate(node_disk_writes_completed_total[1m]) > 0.1 and - rate(node_disk_writes_completed_total[1m]) > 0) * on(instance) group_left (nodename) - node_uname_info{nodename=~".+"}' + '((rate(node_disk_write_time_seconds_total[1m]) / + rate(node_disk_writes_completed_total[1m]) > 0.1) and + rate(node_disk_writes_completed_total[1m]) > 5) * on(instance, device) group_left + node_disk_info{rotational="0", device!~"^(dm-|loop|ram|zram|sr|fd).*"} * on(instance) + group_left (nodename) node_uname_info{nodename=~".+"}' for: 2m labels: severity: warning diff --git a/hosts/12kingdoms-youko/homelab-services/monitoring/alert_rules/postgres-exporter.yml b/hosts/12kingdoms-youko/homelab-services/monitoring/alert_rules/postgres-exporter.yml index 9ea6ef5a..381ab873 100644 --- a/hosts/12kingdoms-youko/homelab-services/monitoring/alert_rules/postgres-exporter.yml +++ b/hosts/12kingdoms-youko/homelab-services/monitoring/alert_rules/postgres-exporter.yml @@ -176,9 +176,8 @@ groups: - alert: PostgresqlConfigurationChanged expr: - '{__name__=~"pg_settings_.*"} != ON(__name__, instance) - {__name__=~"pg_settings_([^t]|t[^r]|tr[^a]|tra[^n]|tran[^s]|trans[^a]|transa[^c]|transac[^t]|transact[^i]|transacti[^o]|transactio[^n]|transaction[^_]|transaction_[^r]|transaction_r[^e]|transaction_re[^a]|transaction_rea[^d]|transaction_read[^_]|transaction_read_[^o]|transaction_read_o[^n]|transaction_read_on[^l]|transaction_read_only[^y]).*"} - OFFSET 5m' + 'changes({__name__=~"pg_settings_.*", __name__!="pg_settings_transaction_read_only"}[5m]) + > 0' for: 0m labels: severity: info diff --git a/outputs/x86_64-linux/src/12kingdoms-shoryu.nix b/outputs/x86_64-linux/src/12kingdoms-shoryu.nix index 008047ab..0be1b242 100644 --- a/outputs/x86_64-linux/src/12kingdoms-shoryu.nix +++ b/outputs/x86_64-linux/src/12kingdoms-shoryu.nix @@ -43,8 +43,6 @@ in systemArgs // { inherit tags ssh-user; - # reach the host by IP: its DNS name changes with the hostname - targetHost = myvars.networking.hostsAddr.${name}.ipv4; } ); diff --git a/outputs/x86_64-linux/src/12kingdoms-shushou.nix b/outputs/x86_64-linux/src/12kingdoms-shushou.nix index 5de08569..e66d163b 100644 --- a/outputs/x86_64-linux/src/12kingdoms-shushou.nix +++ b/outputs/x86_64-linux/src/12kingdoms-shushou.nix @@ -43,8 +43,6 @@ in systemArgs // { inherit tags ssh-user; - # reach the host by IP: its DNS name changes with the hostname - targetHost = myvars.networking.hostsAddr.${name}.ipv4; } ); diff --git a/outputs/x86_64-linux/src/12kingdoms-youko.nix b/outputs/x86_64-linux/src/12kingdoms-youko.nix index 518ccb93..447eabd7 100644 --- a/outputs/x86_64-linux/src/12kingdoms-youko.nix +++ b/outputs/x86_64-linux/src/12kingdoms-youko.nix @@ -43,8 +43,6 @@ in systemArgs // { inherit tags ssh-user; - # reach the host by IP: its DNS name changes with the hostname - targetHost = myvars.networking.hostsAddr.${name}.ipv4; } );