From 9b68cfdb8a7b402f1bf86fdcf5675b645cccceea Mon Sep 17 00:00:00 2001 From: Ryan Yin Date: Wed, 21 Oct 2026 23:21:19 +0800 Subject: [PATCH] feat(backup): alert when the last successful backup goes stale The backup unit publishes `restic_backup_last_success_timestamp_seconds` for the node_exporter textfile collector, and a rejected alert now fails the notify unit instead of exiting 0. Assisted-by: DeepSeek Harness 0.2.1-rc.2 (deepseek-flash) --- .../homelab-services/monitoring/README.md | 11 ++++--- .../monitoring/alert_rules/restic-backup.yml | 31 +++++++++++++++++++ modules/nixos/base/monitoring.nix | 17 ++++++++++ modules/nixos/base/restic-backup.nix | 25 ++++++++++++++- 4 files changed, 78 insertions(+), 6 deletions(-) create mode 100644 hosts/12kingdoms-youko/homelab-services/monitoring/alert_rules/restic-backup.yml diff --git a/hosts/12kingdoms-youko/homelab-services/monitoring/README.md b/hosts/12kingdoms-youko/homelab-services/monitoring/README.md index b2810b61..2bc8c438 100644 --- a/hosts/12kingdoms-youko/homelab-services/monitoring/README.md +++ b/hosts/12kingdoms-youko/homelab-services/monitoring/README.md @@ -7,6 +7,12 @@ - [victoria-metrics-k8s-stack/files/rules](https://github.com/VictoriaMetrics/helm-charts/tree/master/charts/victoria-metrics-k8s-stack/files/rules/generated) - Alert Rules & Recording Rules used by kube-prometheus-stack. +## Backup Freshness + +`modules/nixos/base/restic-backup.nix` writes `restic_backup_last_success_timestamp_seconds` to +node_exporter's textfile collector after a successful run. `alert_rules/restic-backup.yml` alerts +when it is older than two days (the schedule is daily) or absent entirely. + ## TODO - **Dead-man switch**: `Watchdog` in `alert_rules/general.yml` always fires (`vector(1)`) but is @@ -14,8 +20,3 @@ healthy one. Route it to an external heartbeat service (healthchecks.io, Uptime Kuma, PagerDuty DeadMansSnitch) that notifies when the heartbeat stops, or run an out-of-band probe against Alertmanager. -- **Backup freshness**: nothing alerts on "no recent successful backup". The systemd `failed` state - only catches crashes, not a job that never ran or a backup that is silently stale. Export the last - successful backup timestamp (the restic rest-server already exposes Go/promhttp metrics; btrfs - snapshot time via the node-exporter textfile collector) and alert when it exceeds the SLO. - Off-host copies are already restic's job (see `BACKUP.md`), so no btrbk `target` is needed. diff --git a/hosts/12kingdoms-youko/homelab-services/monitoring/alert_rules/restic-backup.yml b/hosts/12kingdoms-youko/homelab-services/monitoring/alert_rules/restic-backup.yml new file mode 100644 index 00000000..82662299 --- /dev/null +++ b/hosts/12kingdoms-youko/homelab-services/monitoring/alert_rules/restic-backup.yml @@ -0,0 +1,31 @@ +groups: + - name: restic-backup.rules + rules: + # restic-backup.nix writes this after every fully successful run (backup, + # prune and check). The schedule is daily (02:30 on youko, 01:30 on the + # desktops, each with up to 15m of random delay), so 48h means two runs in + # a row were missed. Only hosts whose node_exporter is up report this. + - alert: ResticBackupStale + expr: time() - restic_backup_last_success_timestamp_seconds > 2 * 24 * 3600 + for: 10m + labels: + severity: warning + annotations: + summary: No successful restic backup on {{ $labels.host }} in 2 days + description: + "The last successful restic backup on {{ $labels.host }} was {{ $value }} seconds ago. + Check `systemctl status restic-backups-homelab.service` and `journalctl -u + restic-backups-homelab` there." + + # Staleness needs a sample to compare against; this covers the case where + # no host has ever published (or every textfile was wiped). + - alert: ResticBackupMetricMissing + expr: absent(restic_backup_last_success_timestamp_seconds) + for: 26h + labels: + severity: warning + annotations: + summary: No restic backup reports a successful run + description: + "restic_backup_last_success_timestamp_seconds is absent on every host, so backup + freshness cannot be evaluated." diff --git a/modules/nixos/base/monitoring.nix b/modules/nixos/base/monitoring.nix index e8a89d02..90b756ff 100644 --- a/modules/nixos/base/monitoring.nix +++ b/modules/nixos/base/monitoring.nix @@ -12,6 +12,11 @@ let # exporters. hostAddr = myvars.networking.hostsAddr.${config.networking.hostName} or { }; lanAddr = if hostAddr.homeOnly or false then null else hostAddr.ipv4 or null; + + # Where node_exporter's textfile collector reads `*.prom` files. Keep in sync + # with modules/nixos/base/restic-backup.nix, which writes the last successful + # backup timestamp here. + textfileDir = "/var/lib/node_exporter/textfile_collector"; in { # node_exporter on every host that has a static LAN IPv4 (see `lanAddr` above) @@ -25,6 +30,9 @@ in enabledCollectors = [ "systemd" "logind" + # Metrics that no exporter can produce itself, e.g. the timestamp of the + # last successful restic backup (written by restic-backup.nix). + "textfile" ]; # use either enabledCollectors or disabledCollectors @@ -46,9 +54,18 @@ in # Note: ^(/|/persistent/) prefix ensures both root-level and # /persistent-prefixed paths (used in NixOS's tmpfs-as-root setup) are excluded. "--collector.filesystem.mount-points-exclude=^(/|/persistent/)(dev|proc|sys|run/credentials/.+|run/user/.+|var/lib/docker/.+|var/lib/containers/.+|var/lib/kubelet/.+|home/ryan/.+)($|/)" + # The collector does not create its directory, and a missing one logs an + # error on every scrape. + "--collector.textfile.directory=${textfileDir}" ]; }; + # Created here so the textfile collector has somewhere to read from before + # the first successful backup writes its metric. + systemd.tmpfiles.rules = lib.mkIf (lanAddr != null) [ + "d ${textfileDir} 0755 root root - -" + ]; + # Drive SMART health (wear, spare, media errors, temperature, bytes written) # on physical hosts. node_exporter's nvme collector only exposes # node_nvme_info and namespace capacity, NOT SMART health, so this is the only diff --git a/modules/nixos/base/restic-backup.nix b/modules/nixos/base/restic-backup.nix index bc398d29..aa6e1bff 100644 --- a/modules/nixos/base/restic-backup.nix +++ b/modules/nixos/base/restic-backup.nix @@ -12,6 +12,10 @@ let # postgres data directory is only crash-consistent). It is deleted right # after, so it is not a second backup - just a consistency step. snapshotPath = "${builtins.dirOf cfg.snapshotSource}/@restic-snapshot"; + + # node_exporter's textfile collector directory, kept in sync with + # modules/nixos/base/monitoring.nix, which points the collector here. + textfileDir = "/var/lib/node_exporter/textfile_collector"; in { # ================================================================== @@ -210,6 +214,21 @@ in RequiresMountsFor = cfg.requiresMountsFor; }; + # Record when the last fully successful run finished, so staleness is + # alertable (hosts/12kingdoms-youko/homelab-services/monitoring/alert_rules/ + # restic-backup.yml). ExecStartPost runs only after every ExecStart + # (backup, forget --prune, check) succeeded, and the rename keeps + # node_exporter from reading a half-written file. + systemd.services.restic-backups-homelab.postStart = '' + ${pkgs.coreutils}/bin/install -d -m 0755 ${textfileDir} + { + echo '# HELP restic_backup_last_success_timestamp_seconds Unix time of the last successful restic backup run.' + echo '# TYPE restic_backup_last_success_timestamp_seconds gauge' + echo "restic_backup_last_success_timestamp_seconds $(${pkgs.coreutils}/bin/date +%s)" + } > ${textfileDir}/restic-backup.prom.tmp + ${pkgs.coreutils}/bin/mv -f ${textfileDir}/restic-backup.prom.tmp ${textfileDir}/restic-backup.prom + ''; + # A failed backup must not be silent: report it to alertmanager, which # routes anything above info severity to telegram. systemd.services.restic-backups-homelab.onFailure = [ "restic-backup-notify.service" ]; @@ -217,8 +236,12 @@ in systemd.services.restic-backup-notify = { description = "Report a failed restic backup"; serviceConfig.Type = "oneshot"; + # `--fail` makes curl exit non-zero on an HTTP 4xx/5xx response: without + # it a rejected alert still ended the unit with status 0, i.e. silently. + # `--max-time` bounds the request so an unresponsive Alertmanager cannot + # hold the unit open. script = '' - ${pkgs.curl}/bin/curl -sS -o /dev/null -X POST ${cfg.alertManagerUrl}/api/v2/alerts \ + ${pkgs.curl}/bin/curl -sS --fail --max-time 10 -o /dev/null -X POST ${cfg.alertManagerUrl}/api/v2/alerts \ -H 'Content-Type: application/json' \ --data "[{ \ \"labels\": { \"alertname\": \"ResticBackupFailed\", \"severity\": \"critical\", \"host\": \"${config.networking.hostName}\" }, \