diff --git a/BACKUP.md b/BACKUP.md index 694d6fd5..fbe490bf 100644 --- a/BACKUP.md +++ b/BACKUP.md @@ -4,12 +4,12 @@ How this fleet is backed up, where the data lives, and how to restore it. ## What protects against what -| Threat | Defence | -| ----------------------------- | -------------------------------------------------------------------- | -| accidental deletion, bad edit | btrbk: local btrfs snapshots | -| disk or host loss | restic: encrypted copies on youko | -| a compromised backup server | desktop repositories use a password youko never holds | -| a compromised host | not defended yet: the immutable offsite copy (planned) is that layer | +| Threat | Defence | +| ----------------------------- | ------------------------------------------------------------------ | +| accidental deletion, bad edit | btrbk: local btrfs snapshots | +| disk or host loss | restic: encrypted copies on youko | +| a compromised backup server | desktop repositories use a password youko never holds | +| a compromised host | not defended yet: the immutable cloud copy (planned) is that layer | ## Layers @@ -47,6 +47,10 @@ Consequences worth remembering: is the only way to restore its data. - restic's own password is never stored inside a backed-up tree: `etc/agenix` and all key material are excluded (see below). +- Recipients follow the secrets repository's rule: every secret is decryptable by the desktops + (`desktop_keys`, which also carries the offline `recovery_key`), so a desktop can edit or rekey + anything and no secret becomes unrecoverable when a host is lost. The desktop repository password + is the one secret the servers do not get. ## Schedule @@ -114,7 +118,7 @@ Restoring a btrbk snapshot (offline; stop writers first): - btrbk: automatic, 7 days with a 2 day minimum. - youko's own restic repository: automatic, `--keep-daily 3 --keep-weekly 2 --keep-monthly 2`. - Desktop restic repositories: automatic, the module defaults - (`--keep-daily 3 --keep-weekly 2 --keep-monthly 2`). Immutability is the offsite copy's job + (`--keep-daily 3 --keep-weekly 2 --keep-monthly 2`). Immutability is the cloud copy's job (planned), not the local server's. ## Verifying @@ -126,6 +130,22 @@ Restoring a btrbk snapshot (offline; stop writers first): - `restic-rest-server`'s log must not print `Invalid htpasswd entry`: that means the client's REST password and the server's htpasswd disagree, and the credentials pair needs rebuilding from one password. +- Metrics: the rest-server exposes the Go/promhttp defaults on loopback (scraped as the + `restic-rest-server` job; `/metrics` is blocked on the public vhost). Repository growth is charted + from `node-exporter`'s filesystem metrics for youko's `/data` mount instead — rest-server itself + has no per-repository size metrics. +- Failures are not silent: a failed `restic-backups-homelab` starts `restic-backup-notify`, which + posts to alertmanager and, above info severity, on to telegram. alertmanager clears it after + `resolve_timeout`. + +## Planned + +- **Restore drill**: actually restore a file from a restic snapshot and from a btrbk snapshot, on a + schedule. An unverified restore is not a backup. +- **Cloud copy**: youko copies the homelab repositories with `restic copy`; desktops copy their own. + The target must not live in the homelab, and should be immutable or versioned (object lock) — that + is the layer which survives a compromised host. +- **`restic check` timer**: periodic integrity verification, on the host that holds the password. ## Components diff --git a/hosts/12kingdoms-youko/default.nix b/hosts/12kingdoms-youko/default.nix index d802b7ea..61971a56 100644 --- a/hosts/12kingdoms-youko/default.nix +++ b/hosts/12kingdoms-youko/default.nix @@ -73,6 +73,10 @@ in dataDir = "/data/backups/rest-server"; privateRepos = true; "htpasswd-file" = config.age.secrets."restic-rest-htpasswd".path; + # expose /metrics for victoriametrics; unauthenticated is fine because the + # server only listens on loopback + prometheus = true; + extraFlags = [ "--prometheus-no-auth" ]; }; # repositories live on the HDD, which has to be mounted first diff --git a/hosts/12kingdoms-youko/homelab-services/caddy.nix b/hosts/12kingdoms-youko/homelab-services/caddy.nix index 549377f2..1b500dfb 100644 --- a/hosts/12kingdoms-youko/homelab-services/caddy.nix +++ b/hosts/12kingdoms-youko/homelab-services/caddy.nix @@ -66,8 +66,17 @@ in # the restic REST server the backup clients push to virtualHosts."restic.writefor.fun".extraConfig = '' ${hostCommonConfig} + # metrics are scraped over loopback only, never through this vhost + @metrics path /metrics + respond @metrics 403 reverse_proxy http://localhost:8000 ''; + # Every other name is not a service on this host: answer 404 instead of + # caddy's empty 200 fallback, which hides typos. + virtualHosts."*.writefor.fun".extraConfig = '' + ${hostCommonConfig} + respond 404 + ''; virtualHosts."transmission.writefor.fun".extraConfig = '' ${hostCommonConfig} encode zstd gzip diff --git a/hosts/12kingdoms-youko/homelab-services/monitoring/victoriametrics.nix b/hosts/12kingdoms-youko/homelab-services/monitoring/victoriametrics.nix index 1201c2d5..5bfb747a 100644 --- a/hosts/12kingdoms-youko/homelab-services/monitoring/victoriametrics.nix +++ b/hosts/12kingdoms-youko/homelab-services/monitoring/victoriametrics.nix @@ -77,6 +77,22 @@ in } */ + { + job_name = "restic-rest-server"; + scrape_interval = "60s"; + metrics_path = "/metrics"; + static_configs = [ + { + # same-host rest-server bound to loopback (127.0.0.1:8000) + targets = [ "127.0.0.1:8000" ]; + labels.type = "app"; + labels.app = "restic"; + labels.host = "youko"; + labels.env = "homelab"; + labels.cluster = "homelab"; + } + ]; + } { job_name = "v2ray-exporter"; scrape_interval = "30s"; diff --git a/modules/nixos/base/restic-backup.nix b/modules/nixos/base/restic-backup.nix index 987ef40f..e1b6847d 100644 --- a/modules/nixos/base/restic-backup.nix +++ b/modules/nixos/base/restic-backup.nix @@ -109,6 +109,16 @@ in description = "Extra random delay on top of {option}`onCalendar`, to smooth load."; }; + alertManagerUrl = lib.mkOption { + type = lib.types.str; + default = "https://alertmanager.writefor.fun"; + description = '' + Alertmanager to POST a failure alert to (`/api/v2/alerts`). youko routes + anything above info severity to telegram, so a failed backup is not + silent. + ''; + }; + pruneOpts = lib.mkOption { type = lib.types.listOf lib.types.str; default = [ @@ -183,5 +193,22 @@ in { RequiresMountsFor = cfg.requiresMountsFor; }; + + # A failed backup must not be silent: report it to alertmanager, which + # routes anything above info severity to telegram. + systemd.services.restic-backups-homelab.onFailure = [ "restic-backup-notify.service" ]; + + systemd.services.restic-backup-notify = { + description = "Report a failed restic backup"; + serviceConfig.Type = "oneshot"; + script = '' + ${pkgs.curl}/bin/curl -sS -o /dev/null -X POST ${cfg.alertManagerUrl}/api/v2/alerts \ + -H 'Content-Type: application/json' \ + --data "[{ \ + \"labels\": { \"alertname\": \"ResticBackupFailed\", \"severity\": \"critical\", \"host\": \"${config.networking.hostName}\" }, \ + \"annotations\": { \"summary\": \"restic backup failed on ${config.networking.hostName}\", \"description\": \"journalctl -u restic-backups-homelab\" } \ + }]" + ''; + }; }; } diff --git a/secrets/README.md b/secrets/README.md index 6014b2de..6a1a5abe 100644 --- a/secrets/README.md +++ b/secrets/README.md @@ -20,6 +20,20 @@ repository. This directory contains this `README.md`, and a `nixos.nix`/`darwin.nix` file that is used to decrypt all my secrets via `agenix`. Then, I can use them in this flake. +## Which Keys Go on a Secret + +**Every secret is decryptable by the desktops and by the offline `recovery_key`, and those two have +identical access.** The desktops hold the trusted admin keys and are where secrets are added, edited +and rekeyed — including `agenix -r`, which has to decrypt every secret in this repository first. +`recovery_key` is a member of `desktop_keys`, so one rule covers both: every recipient set is +`desktop_keys ++ `. + +That also means a lost host never makes a secret unrecoverable, and that narrowing a set to the +servers alone breaks the desktops and the recovery key together. + +The one exception is the desktop's own restic repository password (`restic-password-desktop.age`): +`desktop_keys` alone, because the backup servers must not be able to read desktop data. + ## Adding or Updating Secrets > All the operations in this section should be performed in my private repository: `nix-secrets`.