From ce49b03014838211c93048a3aa2687f9785c7f1a Mon Sep 17 00:00:00 2001 From: Ryan Yin Date: Mon, 2 Nov 2026 00:59:26 +0800 Subject: [PATCH] fix(monitoring): socket-activate LAN exporters with FreeBind Exporters raced networkd for the LAN address at boot; drop the network-online ordering and unlimited-restart hack and let a FreeBind socket unit hold the port, passed in via --web.systemd-socket. Assisted-by: Claude Code (claude-opus-5-5) --- modules/nixos/base/monitoring.nix | 37 ++++++++++--------- .../tests/security-exporters/expr.nix | 29 +++++++++++---- 2 files changed, 40 insertions(+), 26 deletions(-) diff --git a/modules/nixos/base/monitoring.nix b/modules/nixos/base/monitoring.nix index 2faa48aa..27c61250 100644 --- a/modules/nixos/base/monitoring.nix +++ b/modules/nixos/base/monitoring.nix @@ -17,6 +17,10 @@ let # with modules/nixos/base/restic-backup.nix, which writes the last successful # backup timestamp here. textfileDir = "/var/lib/node_exporter/textfile_collector"; + + lanExporters = lib.filterAttrs (_: cfg: cfg.enable) { + inherit (config.services.prometheus.exporters) node smartctl nvidia-gpu; + }; in { # node_exporter on every host that has a static LAN IPv4 (see `lanAddr` above) @@ -57,6 +61,7 @@ in # The collector does not create its directory, and a missing one logs an # error on every scrape. "--collector.textfile.directory=${textfileDir}" + "--web.systemd-socket" ]; }; @@ -78,25 +83,21 @@ in listenAddress = lanAddr; port = 9633; maxInterval = "60s"; + extraFlags = [ "--web.systemd-socket" ]; }; - # LAN-bound exporters race networkd's address assignment at boot; without - # ordering they trip the start limit (seen on ai). wait-online is best-effort - # (120s timeout), so also retry patiently. - systemd.services = lib.mkIf (lanAddr != null) ( - lib.genAttrs - ( - [ "prometheus-node-exporter" ] - ++ lib.optional config.services.prometheus.exporters.nvidia-gpu.enable "prometheus-nvidia-gpu-exporter" - ++ lib.optionals config.modules.hardwareTools.enable [ "prometheus-smartctl-exporter" ] - ) - (_: { - after = [ "network-online.target" ]; - wants = [ "network-online.target" ]; - serviceConfig = { - RestartSec = "10s"; - StartLimitIntervalSec = 0; - }; - }) + # Socket activation with FreeBind: the LAN address may not exist yet at boot. + systemd.sockets = lib.mkIf (lanAddr != null) ( + lib.mapAttrs' ( + name: cfg: + lib.nameValuePair "prometheus-${name}-exporter" { + wantedBy = [ "sockets.target" ]; + listenStreams = [ "${cfg.listenAddress}:${toString cfg.port}" ]; + socketConfig.FreeBind = true; + } + ) lanExporters ); + services.prometheus.exporters.nvidia-gpu.extraFlags = lib.mkIf (lanAddr != null) [ + "--web.systemd-socket" + ]; } diff --git a/outputs/x86_64-linux/tests/security-exporters/expr.nix b/outputs/x86_64-linux/tests/security-exporters/expr.nix index e6c79ea5..79a6544f 100644 --- a/outputs/x86_64-linux/tests/security-exporters/expr.nix +++ b/outputs/x86_64-linux/tests/security-exporters/expr.nix @@ -43,18 +43,31 @@ lib.genAttrs (builtins.attrNames outputs.nixosConfigurations) ( 9835 9633 ]; - # Every enabled exporter must bind this host's static LAN IPv4, not the - # nixpkgs default 0.0.0.0 (audit 2026-10-22 found the nvidia exporter wild). + # Every enabled exporter's socket must bind this host's static LAN IPv4, not + # the nixpkgs default 0.0.0.0 (audit 2026-10-22 found the nvidia exporter wild). exportersBoundToLan = let lanAddr = myvars.networking.hostsAddr.${config.networking.hostName}.ipv4 or null; - exporters = [ - config.services.prometheus.exporters.node - config.services.prometheus.exporters.smartctl - config.services.prometheus.exporters.nvidia-gpu - ]; in - lib.all (e: !e.enable || e.listenAddress == lanAddr) exporters; + lib.all + ( + name: + let + e = config.services.prometheus.exporters.${name}; + socket = config.systemd.sockets."prometheus-${name}-exporter" or { }; + in + !e.enable + || ( + builtins.elem "--web.systemd-socket" e.extraFlags + && socket.listenStreams or [ ] == [ "${lanAddr}:${toString e.port}" ] + && socket.socketConfig.FreeBind or false + ) + ) + [ + "node" + "smartctl" + "nvidia-gpu" + ]; ipv6RestrictedBeforeTrust = lib.hasInfix ipv6Guard beforeTrust; generatedRestrictionsBeforeTrust = broadTrustRules != [ ]