diff --git a/BACKUP.md b/BACKUP.md index 694d6fd5..775be7d2 100644 --- a/BACKUP.md +++ b/BACKUP.md @@ -126,6 +126,22 @@ Restoring a btrbk snapshot (offline; stop writers first): - `restic-rest-server`'s log must not print `Invalid htpasswd entry`: that means the client's REST password and the server's htpasswd disagree, and the credentials pair needs rebuilding from one password. +- Failures are not silent: a failed `restic-backups-homelab` starts `restic-backup-notify`, which + posts to alertmanager and, above info severity, on to telegram. alertmanager clears it after + `resolve_timeout`. + +## Planned + +- **Restore drill**: actually restore a file from a restic snapshot and from a btrbk snapshot, on a + schedule. An unverified restore is not a backup. +- **Offsite copy**: youko copies the homelab repositories with `restic copy`; desktops copy their + own. Use an immutable or versioned target (object lock) — that is the layer which survives a + compromised host. +- **`restic check` timer**: periodic integrity verification, on the host that holds the password. +- **Metrics**: run rest-server with `--prometheus` and chart repository growth in Grafana. +- **Recovery key for the homelab password**: `restic-password-homelab.age` is encrypted only to the + storage hosts' keys, so losing youko's host key makes it unrecoverable. Adding the offline + `recovery_key` to that group fixes it without weakening the desktop/homelab split. ## Components diff --git a/hosts/12kingdoms-youko/homelab-services/caddy.nix b/hosts/12kingdoms-youko/homelab-services/caddy.nix index 549377f2..e9cb8c22 100644 --- a/hosts/12kingdoms-youko/homelab-services/caddy.nix +++ b/hosts/12kingdoms-youko/homelab-services/caddy.nix @@ -68,6 +68,12 @@ in ${hostCommonConfig} reverse_proxy http://localhost:8000 ''; + # Every other name is not a service on this host: answer 404 instead of + # caddy's empty 200 fallback, which hides typos. + virtualHosts."*.writefor.fun".extraConfig = '' + ${hostCommonConfig} + respond 404 + ''; virtualHosts."transmission.writefor.fun".extraConfig = '' ${hostCommonConfig} encode zstd gzip diff --git a/modules/nixos/base/restic-backup.nix b/modules/nixos/base/restic-backup.nix index 987ef40f..e1b6847d 100644 --- a/modules/nixos/base/restic-backup.nix +++ b/modules/nixos/base/restic-backup.nix @@ -109,6 +109,16 @@ in description = "Extra random delay on top of {option}`onCalendar`, to smooth load."; }; + alertManagerUrl = lib.mkOption { + type = lib.types.str; + default = "https://alertmanager.writefor.fun"; + description = '' + Alertmanager to POST a failure alert to (`/api/v2/alerts`). youko routes + anything above info severity to telegram, so a failed backup is not + silent. + ''; + }; + pruneOpts = lib.mkOption { type = lib.types.listOf lib.types.str; default = [ @@ -183,5 +193,22 @@ in { RequiresMountsFor = cfg.requiresMountsFor; }; + + # A failed backup must not be silent: report it to alertmanager, which + # routes anything above info severity to telegram. + systemd.services.restic-backups-homelab.onFailure = [ "restic-backup-notify.service" ]; + + systemd.services.restic-backup-notify = { + description = "Report a failed restic backup"; + serviceConfig.Type = "oneshot"; + script = '' + ${pkgs.curl}/bin/curl -sS -o /dev/null -X POST ${cfg.alertManagerUrl}/api/v2/alerts \ + -H 'Content-Type: application/json' \ + --data "[{ \ + \"labels\": { \"alertname\": \"ResticBackupFailed\", \"severity\": \"critical\", \"host\": \"${config.networking.hostName}\" }, \ + \"annotations\": { \"summary\": \"restic backup failed on ${config.networking.hostName}\", \"description\": \"journalctl -u restic-backups-homelab\" } \ + }]" + ''; + }; }; }