fix(monitoring): socket-activate LAN exporters with FreeBind

Exporters raced networkd for the LAN address at boot; drop the
network-online ordering and unlimited-restart hack and let a FreeBind
socket unit hold the port, passed in via --web.systemd-socket.

Assisted-by: Claude Code (claude-opus-5-5)
This commit is contained in:
Ryan Yin authored and ryan4yin | 二花 committed 2026-11-01 11:55:14 -06:00
1 parent b063feca29
commit ce49b03014
2 files changed
+40 -26

No files matched your search

+19 -18
View File
@@ -17,6 +17,10 @@ let
# with modules/nixos/base/restic-backup.nix, which writes the last successful
# backup timestamp here.
textfileDir = "/var/lib/node_exporter/textfile_collector";
lanExporters = lib.filterAttrs (_: cfg: cfg.enable) {
inherit (config.services.prometheus.exporters) node smartctl nvidia-gpu;
};
in
{
# node_exporter on every host that has a static LAN IPv4 (see `lanAddr` above)
@@ -57,6 +61,7 @@ in
# The collector does not create its directory, and a missing one logs an
# error on every scrape.
"--collector.textfile.directory=${textfileDir}"
"--web.systemd-socket"
];
};
@@ -78,25 +83,21 @@ in
listenAddress = lanAddr;
port = 9633;
maxInterval = "60s";
extraFlags = [ "--web.systemd-socket" ];
};
# LAN-bound exporters race networkd's address assignment at boot; without
# ordering they trip the start limit (seen on ai). wait-online is best-effort
# (120s timeout), so also retry patiently.
systemd.services = lib.mkIf (lanAddr != null) (
lib.genAttrs
(
[ "prometheus-node-exporter" ]
++ lib.optional config.services.prometheus.exporters.nvidia-gpu.enable "prometheus-nvidia-gpu-exporter"
++ lib.optionals config.modules.hardwareTools.enable [ "prometheus-smartctl-exporter" ]
)
(_: {
after = [ "network-online.target" ];
wants = [ "network-online.target" ];
serviceConfig = {
RestartSec = "10s";
StartLimitIntervalSec = 0;
};
})
# Socket activation with FreeBind: the LAN address may not exist yet at boot.
systemd.sockets = lib.mkIf (lanAddr != null) (
lib.mapAttrs' (
name: cfg:
lib.nameValuePair "prometheus-${name}-exporter" {
wantedBy = [ "sockets.target" ];
listenStreams = [ "${cfg.listenAddress}:${toString cfg.port}" ];
socketConfig.FreeBind = true;
}
) lanExporters
);
services.prometheus.exporters.nvidia-gpu.extraFlags = lib.mkIf (lanAddr != null) [
"--web.systemd-socket"
];
}
@@ -43,18 +43,31 @@ lib.genAttrs (builtins.attrNames outputs.nixosConfigurations) (
9835
9633
];
# Every enabled exporter must bind this host's static LAN IPv4, not the
# nixpkgs default 0.0.0.0 (audit 2026-10-22 found the nvidia exporter wild).
# Every enabled exporter's socket must bind this host's static LAN IPv4, not
# the nixpkgs default 0.0.0.0 (audit 2026-10-22 found the nvidia exporter wild).
exportersBoundToLan =
let
lanAddr = myvars.networking.hostsAddr.${config.networking.hostName}.ipv4 or null;
exporters = [
config.services.prometheus.exporters.node
config.services.prometheus.exporters.smartctl
config.services.prometheus.exporters.nvidia-gpu
];
in
lib.all (e: !e.enable || e.listenAddress == lanAddr) exporters;
lib.all
(
name:
let
e = config.services.prometheus.exporters.${name};
socket = config.systemd.sockets."prometheus-${name}-exporter" or { };
in
!e.enable
|| (
builtins.elem "--web.systemd-socket" e.extraFlags
&& socket.listenStreams or [ ] == [ "${lanAddr}:${toString e.port}" ]
&& socket.socketConfig.FreeBind or false
)
)
[
"node"
"smartctl"
"nvidia-gpu"
];
ipv6RestrictedBeforeTrust = lib.hasInfix ipv6Guard beforeTrust;
generatedRestrictionsBeforeTrust =
broadTrustRules != [ ]