starred/nix-config
1
0
Fork 0
mirror of https://github.com/ryan4yin/nix-config.git synced 2026-08-03 18:28:37 +02:00
Code Issues Packages Projects Releases 10 Wiki Activity

chore(monitoring): stop scraping offline hosts, mute noisy alert levels

Browse Source 
- Add an offlineHosts list to VictoriaMetrics scrape generation: SBCs
  and the whole k3s-prod-1 cluster are powered off, shoukei's exporter
  is disabled on the machine; also comment out the dnsmasq-exporter
  job (runs on suzi).
- Disable node-exporter on shoukei: a laptop on untrusted networks
  should not expose :9100 (the firewall is off repo-wide).
- Route severity none|info alerts to the null receiver; meta alerts
  and info noise should never page.

Signed-off-by: Ryan Yin <xiaoyin_c@qq.com>
This commit is contained in:
Ryan Yin
2026-07-26 22:19:56 +08:00
parent ec208ab4d1
commit 74598b6f6b
3 changed files with 74 additions and 39 deletions
Download Patch File Download Diff File Expand all files Collapse all files
hosts/12kingdoms-shoukei/default.nix
+5
View File
@@ -21,6 +21,11 @@ in
services.sunshine.enable = lib.mkForce false;
services.tuned.ppdSettings.main.default = lib.mkForce "power-saver";
# This laptop joins untrusted networks and the firewall is disabled
# (modules/nixos/base/ssh.nix), so a 0.0.0.0:9100 node-exporter would be exposed
# to whatever network it is on. It is no longer scraped anyway - disable it.
services.prometheus.exporters.node.enable = lib.mkForce false;
networking = {
inherit hostName;
inherit (myvars.networking) nameservers;
hosts/idols-aquamarine/monitoring/alert.nix
+3 -3
View File
@@ -52,10 +52,10 @@
receiver = "telegram";
routes = [
{
# Watchdog & InfoInhibitor are meta alerts that should never notify,
# route them to a null receiver.
# Meta alerts like Watchdog & InfoInhibitor should never notify,
# and info-level alerts are too noisy; route them to a null receiver.
receiver = "null";
matchers = [ ''severity = "none"'' ];
matchers = [ ''severity =~ "none|info"'' ];
}
{
receiver = "telegram";
hosts/idols-aquamarine/monitoring/victoriametrics.nix
+66 -36
View File
@@ -3,6 +3,29 @@
myvars,
...
}:
let
# Hosts that should not be scraped:
# - powered-off machines (SBCs and the whole k3s-prod-1 cluster), to avoid TargetDown
# noise; remove entries from this list when the machines come back online.
# - shoukei (a laptop on untrusted networks), its node-exporter is disabled on the
# machine itself, so there is nothing to scrape.
offlineHosts = [
"shoukei"
"suzu"
"suzi"
"yukina"
"nozomi"
"chiaya"
"rakushun"
"mitsuha"
"k3s-prod-1-master-1"
"k3s-prod-1-master-2"
"k3s-prod-1-master-3"
"k3s-prod-1-worker-1"
"k3s-prod-1-worker-2"
"k3s-prod-1-worker-3"
];
in
{
# Since victoriametrics use DynamicUser, the user & group do not exists before the service starts.
# this group is used as a supplementary Unix group for the service to access our data dir(/data/apps/xxx)
@@ -40,21 +63,24 @@
scrape_configs = [
# --- Homelab Applications --- #
{
job_name = "dnsmasq-exporter";
scrape_interval = "30s";
metrics_path = "/metrics";
static_configs = [
{
targets = [ "${myvars.networking.hostsAddr.suzi.ipv4}:9153" ];
labels.type = "app";
labels.app = "dnsmasq";
labels.host = "suzi";
labels.env = "homelab";
labels.cluster = "homelab";
}
];
}
# suzi is powered off, disable scraping until it comes back online.
/*
{
job_name = "dnsmasq-exporter";
scrape_interval = "30s";
metrics_path = "/metrics";
static_configs = [
{
targets = [ "${myvars.networking.hostsAddr.suzi.ipv4}:9153" ];
labels.type = "app";
labels.app = "dnsmasq";
labels.host = "suzi";
labels.env = "homelab";
labels.cluster = "homelab";
}
];
}
*/
{
job_name = "v2ray-exporter";
@@ -150,27 +176,31 @@
}
]
# --- Hosts --- #
++ (lib.attrsets.foldlAttrs (
acc: hostname: addr:
acc
++ [
{
job_name = "node-exporter-${hostname}";
scrape_interval = "30s";
metrics_path = "/metrics";
static_configs = [
{
# All my NixOS hosts.
targets = [ "${addr.ipv4}:9100" ];
labels.type = "node";
labels.host = hostname;
labels.env = "homelab";
labels.cluster = "homelab";
}
];
}
]
) [ ] myvars.networking.hostsAddr);
++ (lib.attrsets.foldlAttrs
(
acc: hostname: addr:
acc
++ [
{
job_name = "node-exporter-${hostname}";
scrape_interval = "30s";
metrics_path = "/metrics";
static_configs = [
{
# All my NixOS hosts.
targets = [ "${addr.ipv4}:9100" ];
labels.type = "node";
labels.host = hostname;
labels.env = "homelab";
labels.cluster = "homelab";
}
];
}
]
)
[ ]
(lib.attrsets.filterAttrs (n: _: !(builtins.elem n offlineHosts)) myvars.networking.hostsAddr)
);
};
};
}