mirror of
https://github.com/ryan4yin/nix-config.git
synced 2026-11-13 12:13:36 +01:00
fix(monitoring): socket-activate LAN exporters with FreeBind
Exporters raced networkd for the LAN address at boot; drop the network-online ordering and unlimited-restart hack and let a FreeBind socket unit hold the port, passed in via --web.systemd-socket. Assisted-by: Claude Code (claude-opus-5-5)
This commit is contained in:
1 parent
b063feca29
commit
ce49b03014
2 files changed
+40
-26
No files matched your search
@@ -17,6 +17,10 @@ let
|
||||
# with modules/nixos/base/restic-backup.nix, which writes the last successful
|
||||
# backup timestamp here.
|
||||
textfileDir = "/var/lib/node_exporter/textfile_collector";
|
||||
|
||||
lanExporters = lib.filterAttrs (_: cfg: cfg.enable) {
|
||||
inherit (config.services.prometheus.exporters) node smartctl nvidia-gpu;
|
||||
};
|
||||
in
|
||||
{
|
||||
# node_exporter on every host that has a static LAN IPv4 (see `lanAddr` above)
|
||||
@@ -57,6 +61,7 @@ in
|
||||
# The collector does not create its directory, and a missing one logs an
|
||||
# error on every scrape.
|
||||
"--collector.textfile.directory=${textfileDir}"
|
||||
"--web.systemd-socket"
|
||||
];
|
||||
};
|
||||
|
||||
@@ -78,25 +83,21 @@ in
|
||||
listenAddress = lanAddr;
|
||||
port = 9633;
|
||||
maxInterval = "60s";
|
||||
extraFlags = [ "--web.systemd-socket" ];
|
||||
};
|
||||
|
||||
# LAN-bound exporters race networkd's address assignment at boot; without
|
||||
# ordering they trip the start limit (seen on ai). wait-online is best-effort
|
||||
# (120s timeout), so also retry patiently.
|
||||
systemd.services = lib.mkIf (lanAddr != null) (
|
||||
lib.genAttrs
|
||||
(
|
||||
[ "prometheus-node-exporter" ]
|
||||
++ lib.optional config.services.prometheus.exporters.nvidia-gpu.enable "prometheus-nvidia-gpu-exporter"
|
||||
++ lib.optionals config.modules.hardwareTools.enable [ "prometheus-smartctl-exporter" ]
|
||||
)
|
||||
(_: {
|
||||
after = [ "network-online.target" ];
|
||||
wants = [ "network-online.target" ];
|
||||
serviceConfig = {
|
||||
RestartSec = "10s";
|
||||
StartLimitIntervalSec = 0;
|
||||
};
|
||||
})
|
||||
# Socket activation with FreeBind: the LAN address may not exist yet at boot.
|
||||
systemd.sockets = lib.mkIf (lanAddr != null) (
|
||||
lib.mapAttrs' (
|
||||
name: cfg:
|
||||
lib.nameValuePair "prometheus-${name}-exporter" {
|
||||
wantedBy = [ "sockets.target" ];
|
||||
listenStreams = [ "${cfg.listenAddress}:${toString cfg.port}" ];
|
||||
socketConfig.FreeBind = true;
|
||||
}
|
||||
) lanExporters
|
||||
);
|
||||
services.prometheus.exporters.nvidia-gpu.extraFlags = lib.mkIf (lanAddr != null) [
|
||||
"--web.systemd-socket"
|
||||
];
|
||||
}
|
||||
@@ -43,18 +43,31 @@ lib.genAttrs (builtins.attrNames outputs.nixosConfigurations) (
|
||||
9835
|
||||
9633
|
||||
];
|
||||
# Every enabled exporter must bind this host's static LAN IPv4, not the
|
||||
# nixpkgs default 0.0.0.0 (audit 2026-10-22 found the nvidia exporter wild).
|
||||
# Every enabled exporter's socket must bind this host's static LAN IPv4, not
|
||||
# the nixpkgs default 0.0.0.0 (audit 2026-10-22 found the nvidia exporter wild).
|
||||
exportersBoundToLan =
|
||||
let
|
||||
lanAddr = myvars.networking.hostsAddr.${config.networking.hostName}.ipv4 or null;
|
||||
exporters = [
|
||||
config.services.prometheus.exporters.node
|
||||
config.services.prometheus.exporters.smartctl
|
||||
config.services.prometheus.exporters.nvidia-gpu
|
||||
];
|
||||
in
|
||||
lib.all (e: !e.enable || e.listenAddress == lanAddr) exporters;
|
||||
lib.all
|
||||
(
|
||||
name:
|
||||
let
|
||||
e = config.services.prometheus.exporters.${name};
|
||||
socket = config.systemd.sockets."prometheus-${name}-exporter" or { };
|
||||
in
|
||||
!e.enable
|
||||
|| (
|
||||
builtins.elem "--web.systemd-socket" e.extraFlags
|
||||
&& socket.listenStreams or [ ] == [ "${lanAddr}:${toString e.port}" ]
|
||||
&& socket.socketConfig.FreeBind or false
|
||||
)
|
||||
)
|
||||
[
|
||||
"node"
|
||||
"smartctl"
|
||||
"nvidia-gpu"
|
||||
];
|
||||
ipv6RestrictedBeforeTrust = lib.hasInfix ipv6Guard beforeTrust;
|
||||
generatedRestrictionsBeforeTrust =
|
||||
broadTrustRules != [ ]
|
||||
|
||||
Reference in new issue
Block a user