feat(backup): alert when the last successful backup goes stale

The backup unit publishes `restic_backup_last_success_timestamp_seconds` for the node_exporter textfile collector, and a rejected alert now fails the notify unit instead of exiting 0.

Assisted-by: DeepSeek Harness 0.2.1-rc.2 (deepseek-flash)

modules/nixos/base/monitoring.nix

@@ -12,6 +12,11 @@ let
   # exporters.
   hostAddr = myvars.networking.hostsAddr.${config.networking.hostName} or { };
   lanAddr = if hostAddr.homeOnly or false then null else hostAddr.ipv4 or null;
+
+  # Where node_exporter's textfile collector reads `*.prom` files. Keep in sync
+  # with modules/nixos/base/restic-backup.nix, which writes the last successful
+  # backup timestamp here.
+  textfileDir = "/var/lib/node_exporter/textfile_collector";
 in
 {
   # node_exporter on every host that has a static LAN IPv4 (see `lanAddr` above)
@@ -25,6 +30,9 @@ in
     enabledCollectors = [
       "systemd"
       "logind"
+      # Metrics that no exporter can produce itself, e.g. the timestamp of the
+      # last successful restic backup (written by restic-backup.nix).
+      "textfile"
     ];
 
     # use either enabledCollectors or disabledCollectors
@@ -46,9 +54,18 @@ in
       # Note: ^(/|/persistent/) prefix ensures both root-level and
       #       /persistent-prefixed paths (used in NixOS's tmpfs-as-root setup) are excluded.
       "--collector.filesystem.mount-points-exclude=^(/|/persistent/)(dev|proc|sys|run/credentials/.+|run/user/.+|var/lib/docker/.+|var/lib/containers/.+|var/lib/kubelet/.+|home/ryan/.+)($|/)"
+      # The collector does not create its directory, and a missing one logs an
+      # error on every scrape.
+      "--collector.textfile.directory=${textfileDir}"
     ];
   };
 
+  # Created here so the textfile collector has somewhere to read from before
+  # the first successful backup writes its metric.
+  systemd.tmpfiles.rules = lib.mkIf (lanAddr != null) [
+    "d ${textfileDir} 0755 root root - -"
+  ];
+
   # Drive SMART health (wear, spare, media errors, temperature, bytes written)
   # on physical hosts. node_exporter's nvme collector only exposes
   # node_nvme_info and namespace capacity, NOT SMART health, so this is the only

modules/nixos/base/restic-backup.nix

@@ -12,6 +12,10 @@ let
   # postgres data directory is only crash-consistent). It is deleted right
   # after, so it is not a second backup - just a consistency step.
   snapshotPath = "${builtins.dirOf cfg.snapshotSource}/@restic-snapshot";
+
+  # node_exporter's textfile collector directory, kept in sync with
+  # modules/nixos/base/monitoring.nix, which points the collector here.
+  textfileDir = "/var/lib/node_exporter/textfile_collector";
 in
 {
   # ==================================================================
@@ -210,6 +214,21 @@ in
           RequiresMountsFor = cfg.requiresMountsFor;
         };
 
+    # Record when the last fully successful run finished, so staleness is
+    # alertable (hosts/12kingdoms-youko/homelab-services/monitoring/alert_rules/
+    # restic-backup.yml). ExecStartPost runs only after every ExecStart
+    # (backup, forget --prune, check) succeeded, and the rename keeps
+    # node_exporter from reading a half-written file.
+    systemd.services.restic-backups-homelab.postStart = ''
+      ${pkgs.coreutils}/bin/install -d -m 0755 ${textfileDir}
+      {
+        echo '# HELP restic_backup_last_success_timestamp_seconds Unix time of the last successful restic backup run.'
+        echo '# TYPE restic_backup_last_success_timestamp_seconds gauge'
+        echo "restic_backup_last_success_timestamp_seconds $(${pkgs.coreutils}/bin/date +%s)"
+      } > ${textfileDir}/restic-backup.prom.tmp
+      ${pkgs.coreutils}/bin/mv -f ${textfileDir}/restic-backup.prom.tmp ${textfileDir}/restic-backup.prom
+    '';
+
     # A failed backup must not be silent: report it to alertmanager, which
     # routes anything above info severity to telegram.
     systemd.services.restic-backups-homelab.onFailure = [ "restic-backup-notify.service" ];
@@ -217,8 +236,12 @@ in
     systemd.services.restic-backup-notify = {
       description = "Report a failed restic backup";
       serviceConfig.Type = "oneshot";
+      # `--fail` makes curl exit non-zero on an HTTP 4xx/5xx response: without
+      # it a rejected alert still ended the unit with status 0, i.e. silently.
+      # `--max-time` bounds the request so an unresponsive Alertmanager cannot
+      # hold the unit open.
       script = ''
-        ${pkgs.curl}/bin/curl -sS -o /dev/null -X POST ${cfg.alertManagerUrl}/api/v2/alerts \
+        ${pkgs.curl}/bin/curl -sS --fail --max-time 10 -o /dev/null -X POST ${cfg.alertManagerUrl}/api/v2/alerts \
           -H 'Content-Type: application/json' \
           --data "[{ \
             \"labels\": { \"alertname\": \"ResticBackupFailed\", \"severity\": \"critical\", \"host\": \"${config.networking.hostName}\" }, \