fix(monitoring): restore youko metrics, alerts and VM-host deploys (#431)

* flake.lock: Update

nixpkgs only. The new revision builds VictoriaMetrics with cgo
(CGO_ENABLED=1); its pure-Go zstd decoder rejected a stored values block, so
every small-part merge panicked on youko.

* fix(colmena): address the VM hosts by name, not by their IP

`~/.ssh/config` pins the passphrase-protected builder key with IdentitiesOnly
for 192.168.*, so IP-addressed ssh cannot authenticate without a terminal. The
name resolves through the aliases in /etc/ssh/ssh_config.

* fix(monitoring): repair two alert rules VictoriaMetrics rejects

`on(uid)` matches on a label kube-state-metrics no longer emits, and
`!= ON(__name__, instance)` treats every pg_settings series as a duplicate.
They now join on (namespace, pod) and compare with `changes(...[5m])`.

* fix(monitoring): keep disk latency alerts off mechanical disks

Both rules averaged per-op latency with no device or sample guard, so the 4 TB
USB HDD volumes paged constantly (>100 ms/write in 26% of the last 30 days).
They now require a non-rotational physical device and >=5 ops/s.

Assisted-by: DeepSeek Harness 0.2.0-rc.2 (deepseek-flash)

flake.lock

@@ -974,11 +974,11 @@
     },
     "nixpkgs_2": {
       "locked": {
-        "lastModified": 1790822859,
-        "narHash": "sha256-69xHQhAeMAD2wDXO7T2pcOZIF9Sga2W+JkmY2a11Ops=",
-        "owner": "nixos",
+        "lastModified": 1791000213,
+        "narHash": "sha256-qsAQkushLI1nYI/Uz/OAufjcvg50bZBGBq25pe/jcWk=",
+        "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "c59305bab2065cfecc4944690d9eedbb56f3a9fa",
+        "rev": "f71476e8faba8240824d77877d2da77cc42e6d00",
         "type": "github"
       },
       "original": {

hosts/12kingdoms-youko/homelab-services/monitoring/alert_rules/kube-state-metrics-exporter.yml

@@ -48,8 +48,8 @@ groups:
 
       - alert: KubernetesNodeOutOfPodCapacity
         expr:
-          'sum by (node) ((kube_pod_status_phase{phase="Running"} == 1) + on(uid) group_left(node)
-          (0 * kube_pod_info{pod_template_hash=""})) / sum by (node)
+          'sum by (node) ((kube_pod_status_phase{phase="Running"} == 1) + on(namespace, pod)
+          group_left(node) (0 * kube_pod_info{pod_template_hash=""})) / sum by (node)
           (kube_node_status_allocatable{resource="pods"}) * 100 > 90'
         for: 2m
         labels:

hosts/12kingdoms-youko/homelab-services/monitoring/alert_rules/node-exporter.yml

@@ -166,9 +166,10 @@ groups:
 
       - alert: HostUnusualDiskReadLatency
         expr:
-          '(rate(node_disk_read_time_seconds_total[1m]) / rate(node_disk_reads_completed_total[1m])
-          > 0.1 and rate(node_disk_reads_completed_total[1m]) > 0) * on(instance) group_left
-          (nodename) node_uname_info{nodename=~".+"}'
+          '((rate(node_disk_read_time_seconds_total[1m]) / rate(node_disk_reads_completed_total[1m])
+          > 0.1) and rate(node_disk_reads_completed_total[1m]) > 5) * on(instance, device)
+          group_left node_disk_info{rotational="0", device!~"^(dm-|loop|ram|zram|sr|fd).*"} *
+          on(instance) group_left (nodename) node_uname_info{nodename=~".+"}'
         for: 2m
         labels:
           severity: warning
@@ -180,10 +181,11 @@ groups:
 
       - alert: HostUnusualDiskWriteLatency
         expr:
-          '(rate(node_disk_write_time_seconds_total[1m]) /
-          rate(node_disk_writes_completed_total[1m]) > 0.1 and
-          rate(node_disk_writes_completed_total[1m]) > 0) * on(instance) group_left (nodename)
-          node_uname_info{nodename=~".+"}'
+          '((rate(node_disk_write_time_seconds_total[1m]) /
+          rate(node_disk_writes_completed_total[1m]) > 0.1) and
+          rate(node_disk_writes_completed_total[1m]) > 5) * on(instance, device) group_left
+          node_disk_info{rotational="0", device!~"^(dm-|loop|ram|zram|sr|fd).*"} * on(instance)
+          group_left (nodename) node_uname_info{nodename=~".+"}'
         for: 2m
         labels:
           severity: warning

hosts/12kingdoms-youko/homelab-services/monitoring/alert_rules/postgres-exporter.yml

@@ -176,9 +176,8 @@ groups:
 
       - alert: PostgresqlConfigurationChanged
         expr:
-          '{__name__=~"pg_settings_.*"} != ON(__name__, instance)
-          {__name__=~"pg_settings_([^t]|t[^r]|tr[^a]|tra[^n]|tran[^s]|trans[^a]|transa[^c]|transac[^t]|transact[^i]|transacti[^o]|transactio[^n]|transaction[^_]|transaction_[^r]|transaction_r[^e]|transaction_re[^a]|transaction_rea[^d]|transaction_read[^_]|transaction_read_[^o]|transaction_read_o[^n]|transaction_read_on[^l]|transaction_read_only[^y]).*"}
-          OFFSET 5m'
+          'changes({__name__=~"pg_settings_.*", __name__!="pg_settings_transaction_read_only"}[5m])
+          > 0'
         for: 0m
         labels:
           severity: info

outputs/x86_64-linux/src/12kingdoms-shoryu.nix

@@ -43,8 +43,6 @@ in
     systemArgs
     // {
       inherit tags ssh-user;
-      # reach the host by IP: its DNS name changes with the hostname
-      targetHost = myvars.networking.hostsAddr.${name}.ipv4;
     }
   );
 

outputs/x86_64-linux/src/12kingdoms-shushou.nix

@@ -43,8 +43,6 @@ in
     systemArgs
     // {
       inherit tags ssh-user;
-      # reach the host by IP: its DNS name changes with the hostname
-      targetHost = myvars.networking.hostsAddr.${name}.ipv4;
     }
   );
 

outputs/x86_64-linux/src/12kingdoms-youko.nix

@@ -43,8 +43,6 @@ in
     systemArgs
     // {
       inherit tags ssh-user;
-      # reach the host by IP: its DNS name changes with the hostname
-      targetHost = myvars.networking.hostsAddr.${name}.ipv4;
     }
   );