chore(monitoring): stop scraping offline hosts, mute noisy alert levels

- Add an offlineHosts list to VictoriaMetrics scrape generation: SBCs
  and the whole k3s-prod-1 cluster are powered off, shoukei's exporter
  is disabled on the machine; also comment out the dnsmasq-exporter
  job (runs on suzi).
- Disable node-exporter on shoukei: a laptop on untrusted networks
  should not expose :9100 (the firewall is off repo-wide).
- Route severity none|info alerts to the null receiver; meta alerts
  and info noise should never page.

Signed-off-by: Ryan Yin <xiaoyin_c@qq.com>

hosts/12kingdoms-shoukei/default.nix

@@ -21,6 +21,11 @@ in
   services.sunshine.enable = lib.mkForce false;
   services.tuned.ppdSettings.main.default = lib.mkForce "power-saver";
 
+  # This laptop joins untrusted networks and the firewall is disabled
+  # (modules/nixos/base/ssh.nix), so a 0.0.0.0:9100 node-exporter would be exposed
+  # to whatever network it is on. It is no longer scraped anyway - disable it.
+  services.prometheus.exporters.node.enable = lib.mkForce false;
+
   networking = {
     inherit hostName;
     inherit (myvars.networking) nameservers;

hosts/idols-aquamarine/monitoring/alert.nix

@@ -52,10 +52,10 @@
         receiver = "telegram";
         routes = [
           {
-            # Watchdog & InfoInhibitor are meta alerts that should never notify,
+            # Meta alerts like Watchdog & InfoInhibitor should never notify,
-            # route them to a null receiver.
+            # and info-level alerts are too noisy; route them to a null receiver.
             receiver = "null";
-            matchers = [ ''severity = "none"'' ];
+            matchers = [ ''severity =~ "none|info"'' ];
           }
           {
             receiver = "telegram";

hosts/idols-aquamarine/monitoring/victoriametrics.nix

@@ -3,6 +3,29 @@
   myvars,
   ...
 }:
+let
+  # Hosts that should not be scraped:
+  # - powered-off machines (SBCs and the whole k3s-prod-1 cluster), to avoid TargetDown
+  #   noise; remove entries from this list when the machines come back online.
+  # - shoukei (a laptop on untrusted networks), its node-exporter is disabled on the
+  #   machine itself, so there is nothing to scrape.
+  offlineHosts = [
+    "shoukei"
+    "suzu"
+    "suzi"
+    "yukina"
+    "nozomi"
+    "chiaya"
+    "rakushun"
+    "mitsuha"
+    "k3s-prod-1-master-1"
+    "k3s-prod-1-master-2"
+    "k3s-prod-1-master-3"
+    "k3s-prod-1-worker-1"
+    "k3s-prod-1-worker-2"
+    "k3s-prod-1-worker-3"
+  ];
+in
 {
   # Since victoriametrics use DynamicUser, the user & group do not exists before the service starts.
   # this group is used as a supplementary Unix group for the service to access our data dir(/data/apps/xxx)
@@ -40,21 +63,24 @@
       scrape_configs = [
         # --- Homelab Applications --- #
 
-        {
+        # suzi is powered off, disable scraping until it comes back online.
-          job_name = "dnsmasq-exporter";
+        /*
-          scrape_interval = "30s";
+          {
-          metrics_path = "/metrics";
+            job_name = "dnsmasq-exporter";
-          static_configs = [
+            scrape_interval = "30s";
-            {
+            metrics_path = "/metrics";
-              targets = [ "${myvars.networking.hostsAddr.suzi.ipv4}:9153" ];
+            static_configs = [
-              labels.type = "app";
+              {
-              labels.app = "dnsmasq";
+                targets = [ "${myvars.networking.hostsAddr.suzi.ipv4}:9153" ];
-              labels.host = "suzi";
+                labels.type = "app";
-              labels.env = "homelab";
+                labels.app = "dnsmasq";
-              labels.cluster = "homelab";
+                labels.host = "suzi";
-            }
+                labels.env = "homelab";
-          ];
+                labels.cluster = "homelab";
-        }
+              }
+            ];
+          }
+        */
 
         {
           job_name = "v2ray-exporter";
@@ -150,27 +176,31 @@
         }
       ]
       # --- Hosts --- #
-      ++ (lib.attrsets.foldlAttrs (
+      ++ (lib.attrsets.foldlAttrs
-        acc: hostname: addr:
+        (
-        acc
+          acc: hostname: addr:
-        ++ [
+          acc
-          {
+          ++ [
-            job_name = "node-exporter-${hostname}";
+            {
-            scrape_interval = "30s";
+              job_name = "node-exporter-${hostname}";
-            metrics_path = "/metrics";
+              scrape_interval = "30s";
-            static_configs = [
+              metrics_path = "/metrics";
-              {
+              static_configs = [
-                # All my NixOS hosts.
+                {
-                targets = [ "${addr.ipv4}:9100" ];
+                  # All my NixOS hosts.
-                labels.type = "node";
+                  targets = [ "${addr.ipv4}:9100" ];
-                labels.host = hostname;
+                  labels.type = "node";
-                labels.env = "homelab";
+                  labels.host = hostname;
-                labels.cluster = "homelab";
+                  labels.env = "homelab";
-              }
+                  labels.cluster = "homelab";
-            ];
+                }
-          }
+              ];
-        ]
+            }
-      ) [ ] myvars.networking.hostsAddr);
+          ]
+        )
+        [ ]
+        (lib.attrsets.filterAttrs (n: _: !(builtins.elem n offlineHosts)) myvars.networking.hostsAddr)
+      );
     };
   };
 }