feat(backup): alert when the last successful backup goes stale

The backup unit publishes `restic_backup_last_success_timestamp_seconds` for the node_exporter textfile collector, and a rejected alert now fails the notify unit instead of exiting 0.

Assisted-by: DeepSeek Harness 0.2.1-rc.2 (deepseek-flash)

hosts/12kingdoms-youko/homelab-services/monitoring/README.md

@@ -7,6 +7,12 @@
 - [victoria-metrics-k8s-stack/files/rules](https://github.com/VictoriaMetrics/helm-charts/tree/master/charts/victoria-metrics-k8s-stack/files/rules/generated)
   - Alert Rules & Recording Rules used by kube-prometheus-stack.
 
+## Backup Freshness
+
+`modules/nixos/base/restic-backup.nix` writes `restic_backup_last_success_timestamp_seconds` to
+node_exporter's textfile collector after a successful run. `alert_rules/restic-backup.yml` alerts
+when it is older than two days (the schedule is daily) or absent entirely.
+
 ## TODO
 
 - **Dead-man switch**: `Watchdog` in `alert_rules/general.yml` always fires (`vector(1)`) but is
@@ -14,8 +20,3 @@
   healthy one. Route it to an external heartbeat service (healthchecks.io, Uptime Kuma, PagerDuty
   DeadMansSnitch) that notifies when the heartbeat stops, or run an out-of-band probe against
   Alertmanager.
-- **Backup freshness**: nothing alerts on "no recent successful backup". The systemd `failed` state
-  only catches crashes, not a job that never ran or a backup that is silently stale. Export the last
-  successful backup timestamp (the restic rest-server already exposes Go/promhttp metrics; btrfs
-  snapshot time via the node-exporter textfile collector) and alert when it exceeds the SLO.
-  Off-host copies are already restic's job (see `BACKUP.md`), so no btrbk `target` is needed.

hosts/12kingdoms-youko/homelab-services/monitoring/alert_rules/restic-backup.yml

@@ -0,0 +1,31 @@
+groups:
+  - name: restic-backup.rules
+    rules:
+      # restic-backup.nix writes this after every fully successful run (backup,
+      # prune and check). The schedule is daily (02:30 on youko, 01:30 on the
+      # desktops, each with up to 15m of random delay), so 48h means two runs in
+      # a row were missed. Only hosts whose node_exporter is up report this.
+      - alert: ResticBackupStale
+        expr: time() - restic_backup_last_success_timestamp_seconds > 2 * 24 * 3600
+        for: 10m
+        labels:
+          severity: warning
+        annotations:
+          summary: No successful restic backup on {{ $labels.host }} in 2 days
+          description:
+            "The last successful restic backup on {{ $labels.host }} was {{ $value }} seconds ago.
+            Check `systemctl status restic-backups-homelab.service` and `journalctl -u
+            restic-backups-homelab` there."
+
+      # Staleness needs a sample to compare against; this covers the case where
+      # no host has ever published (or every textfile was wiped).
+      - alert: ResticBackupMetricMissing
+        expr: absent(restic_backup_last_success_timestamp_seconds)
+        for: 26h
+        labels:
+          severity: warning
+        annotations:
+          summary: No restic backup reports a successful run
+          description:
+            "restic_backup_last_success_timestamp_seconds is absent on every host, so backup
+            freshness cannot be evaluated."

modules/nixos/base/monitoring.nix

@@ -12,6 +12,11 @@ let
   # exporters.
   hostAddr = myvars.networking.hostsAddr.${config.networking.hostName} or { };
   lanAddr = if hostAddr.homeOnly or false then null else hostAddr.ipv4 or null;
+
+  # Where node_exporter's textfile collector reads `*.prom` files. Keep in sync
+  # with modules/nixos/base/restic-backup.nix, which writes the last successful
+  # backup timestamp here.
+  textfileDir = "/var/lib/node_exporter/textfile_collector";
 in
 {
   # node_exporter on every host that has a static LAN IPv4 (see `lanAddr` above)
@@ -25,6 +30,9 @@ in
     enabledCollectors = [
       "systemd"
       "logind"
+      # Metrics that no exporter can produce itself, e.g. the timestamp of the
+      # last successful restic backup (written by restic-backup.nix).
+      "textfile"
     ];
 
     # use either enabledCollectors or disabledCollectors
@@ -46,9 +54,18 @@ in
       # Note: ^(/|/persistent/) prefix ensures both root-level and
       #       /persistent-prefixed paths (used in NixOS's tmpfs-as-root setup) are excluded.
       "--collector.filesystem.mount-points-exclude=^(/|/persistent/)(dev|proc|sys|run/credentials/.+|run/user/.+|var/lib/docker/.+|var/lib/containers/.+|var/lib/kubelet/.+|home/ryan/.+)($|/)"
+      # The collector does not create its directory, and a missing one logs an
+      # error on every scrape.
+      "--collector.textfile.directory=${textfileDir}"
     ];
   };
 
+  # Created here so the textfile collector has somewhere to read from before
+  # the first successful backup writes its metric.
+  systemd.tmpfiles.rules = lib.mkIf (lanAddr != null) [
+    "d ${textfileDir} 0755 root root - -"
+  ];
+
   # Drive SMART health (wear, spare, media errors, temperature, bytes written)
   # on physical hosts. node_exporter's nvme collector only exposes
   # node_nvme_info and namespace capacity, NOT SMART health, so this is the only

modules/nixos/base/restic-backup.nix

@@ -12,6 +12,10 @@ let
   # postgres data directory is only crash-consistent). It is deleted right
   # after, so it is not a second backup - just a consistency step.
   snapshotPath = "${builtins.dirOf cfg.snapshotSource}/@restic-snapshot";
+
+  # node_exporter's textfile collector directory, kept in sync with
+  # modules/nixos/base/monitoring.nix, which points the collector here.
+  textfileDir = "/var/lib/node_exporter/textfile_collector";
 in
 {
   # ==================================================================
@@ -210,6 +214,21 @@ in
           RequiresMountsFor = cfg.requiresMountsFor;
         };
 
+    # Record when the last fully successful run finished, so staleness is
+    # alertable (hosts/12kingdoms-youko/homelab-services/monitoring/alert_rules/
+    # restic-backup.yml). ExecStartPost runs only after every ExecStart
+    # (backup, forget --prune, check) succeeded, and the rename keeps
+    # node_exporter from reading a half-written file.
+    systemd.services.restic-backups-homelab.postStart = ''
+      ${pkgs.coreutils}/bin/install -d -m 0755 ${textfileDir}
+      {
+        echo '# HELP restic_backup_last_success_timestamp_seconds Unix time of the last successful restic backup run.'
+        echo '# TYPE restic_backup_last_success_timestamp_seconds gauge'
+        echo "restic_backup_last_success_timestamp_seconds $(${pkgs.coreutils}/bin/date +%s)"
+      } > ${textfileDir}/restic-backup.prom.tmp
+      ${pkgs.coreutils}/bin/mv -f ${textfileDir}/restic-backup.prom.tmp ${textfileDir}/restic-backup.prom
+    '';
+
     # A failed backup must not be silent: report it to alertmanager, which
     # routes anything above info severity to telegram.
     systemd.services.restic-backups-homelab.onFailure = [ "restic-backup-notify.service" ];
@@ -217,8 +236,12 @@ in
     systemd.services.restic-backup-notify = {
       description = "Report a failed restic backup";
       serviceConfig.Type = "oneshot";
+      # `--fail` makes curl exit non-zero on an HTTP 4xx/5xx response: without
+      # it a rejected alert still ended the unit with status 0, i.e. silently.
+      # `--max-time` bounds the request so an unresponsive Alertmanager cannot
+      # hold the unit open.
       script = ''
-        ${pkgs.curl}/bin/curl -sS -o /dev/null -X POST ${cfg.alertManagerUrl}/api/v2/alerts \
+        ${pkgs.curl}/bin/curl -sS --fail --max-time 10 -o /dev/null -X POST ${cfg.alertManagerUrl}/api/v2/alerts \
           -H 'Content-Type: application/json' \
           --data "[{ \
             \"labels\": { \"alertname\": \"ResticBackupFailed\", \"severity\": \"critical\", \"host\": \"${config.networking.hostName}\" }, \