From 096607a5c5b8388cea0125f9639da7afb82eba67 Mon Sep 17 00:00:00 2001 From: Ryan Yin Date: Mon, 21 Sep 2026 20:41:30 +0800 Subject: [PATCH 1/7] feat(backup): notify on failure, and 404 unconfigured hostnames A failed backup was silent. Post to alertmanager (which routes above-info to telegram), and have caddy answer 404 instead of its empty 200 for *.writefor.fun names that are not services. --- BACKUP.md | 16 +++++++++++ .../homelab-services/caddy.nix | 6 +++++ modules/nixos/base/restic-backup.nix | 27 +++++++++++++++++++ 3 files changed, 49 insertions(+) diff --git a/BACKUP.md b/BACKUP.md index 694d6fd5..775be7d2 100644 --- a/BACKUP.md +++ b/BACKUP.md @@ -126,6 +126,22 @@ Restoring a btrbk snapshot (offline; stop writers first): - `restic-rest-server`'s log must not print `Invalid htpasswd entry`: that means the client's REST password and the server's htpasswd disagree, and the credentials pair needs rebuilding from one password. +- Failures are not silent: a failed `restic-backups-homelab` starts `restic-backup-notify`, which + posts to alertmanager and, above info severity, on to telegram. alertmanager clears it after + `resolve_timeout`. + +## Planned + +- **Restore drill**: actually restore a file from a restic snapshot and from a btrbk snapshot, on a + schedule. An unverified restore is not a backup. +- **Offsite copy**: youko copies the homelab repositories with `restic copy`; desktops copy their + own. Use an immutable or versioned target (object lock) — that is the layer which survives a + compromised host. +- **`restic check` timer**: periodic integrity verification, on the host that holds the password. +- **Metrics**: run rest-server with `--prometheus` and chart repository growth in Grafana. +- **Recovery key for the homelab password**: `restic-password-homelab.age` is encrypted only to the + storage hosts' keys, so losing youko's host key makes it unrecoverable. Adding the offline + `recovery_key` to that group fixes it without weakening the desktop/homelab split. ## Components diff --git a/hosts/12kingdoms-youko/homelab-services/caddy.nix b/hosts/12kingdoms-youko/homelab-services/caddy.nix index 549377f2..e9cb8c22 100644 --- a/hosts/12kingdoms-youko/homelab-services/caddy.nix +++ b/hosts/12kingdoms-youko/homelab-services/caddy.nix @@ -68,6 +68,12 @@ in ${hostCommonConfig} reverse_proxy http://localhost:8000 ''; + # Every other name is not a service on this host: answer 404 instead of + # caddy's empty 200 fallback, which hides typos. + virtualHosts."*.writefor.fun".extraConfig = '' + ${hostCommonConfig} + respond 404 + ''; virtualHosts."transmission.writefor.fun".extraConfig = '' ${hostCommonConfig} encode zstd gzip diff --git a/modules/nixos/base/restic-backup.nix b/modules/nixos/base/restic-backup.nix index 987ef40f..e1b6847d 100644 --- a/modules/nixos/base/restic-backup.nix +++ b/modules/nixos/base/restic-backup.nix @@ -109,6 +109,16 @@ in description = "Extra random delay on top of {option}`onCalendar`, to smooth load."; }; + alertManagerUrl = lib.mkOption { + type = lib.types.str; + default = "https://alertmanager.writefor.fun"; + description = '' + Alertmanager to POST a failure alert to (`/api/v2/alerts`). youko routes + anything above info severity to telegram, so a failed backup is not + silent. + ''; + }; + pruneOpts = lib.mkOption { type = lib.types.listOf lib.types.str; default = [ @@ -183,5 +193,22 @@ in { RequiresMountsFor = cfg.requiresMountsFor; }; + + # A failed backup must not be silent: report it to alertmanager, which + # routes anything above info severity to telegram. + systemd.services.restic-backups-homelab.onFailure = [ "restic-backup-notify.service" ]; + + systemd.services.restic-backup-notify = { + description = "Report a failed restic backup"; + serviceConfig.Type = "oneshot"; + script = '' + ${pkgs.curl}/bin/curl -sS -o /dev/null -X POST ${cfg.alertManagerUrl}/api/v2/alerts \ + -H 'Content-Type: application/json' \ + --data "[{ \ + \"labels\": { \"alertname\": \"ResticBackupFailed\", \"severity\": \"critical\", \"host\": \"${config.networking.hostName}\" }, \ + \"annotations\": { \"summary\": \"restic backup failed on ${config.networking.hostName}\", \"description\": \"journalctl -u restic-backups-homelab\" } \ + }]" + ''; + }; }; } From 8a44c23d74a5a1602bf0cb4ef743ba63d5707faa Mon Sep 17 00:00:00 2001 From: Ryan Yin Date: Mon, 21 Sep 2026 20:44:56 +0800 Subject: [PATCH 2/7] docs(backup): say 'cloud copy' rather than 'offsite' Clearer for a reader who does not live in backup jargon; the requirement (not in the homelab, immutable) is spelled out in the text. --- BACKUP.md | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/BACKUP.md b/BACKUP.md index 775be7d2..eb5b3478 100644 --- a/BACKUP.md +++ b/BACKUP.md @@ -4,12 +4,12 @@ How this fleet is backed up, where the data lives, and how to restore it. ## What protects against what -| Threat | Defence | -| ----------------------------- | -------------------------------------------------------------------- | -| accidental deletion, bad edit | btrbk: local btrfs snapshots | -| disk or host loss | restic: encrypted copies on youko | -| a compromised backup server | desktop repositories use a password youko never holds | -| a compromised host | not defended yet: the immutable offsite copy (planned) is that layer | +| Threat | Defence | +| ----------------------------- | ------------------------------------------------------------------ | +| accidental deletion, bad edit | btrbk: local btrfs snapshots | +| disk or host loss | restic: encrypted copies on youko | +| a compromised backup server | desktop repositories use a password youko never holds | +| a compromised host | not defended yet: the immutable cloud copy (planned) is that layer | ## Layers @@ -114,7 +114,7 @@ Restoring a btrbk snapshot (offline; stop writers first): - btrbk: automatic, 7 days with a 2 day minimum. - youko's own restic repository: automatic, `--keep-daily 3 --keep-weekly 2 --keep-monthly 2`. - Desktop restic repositories: automatic, the module defaults - (`--keep-daily 3 --keep-weekly 2 --keep-monthly 2`). Immutability is the offsite copy's job + (`--keep-daily 3 --keep-weekly 2 --keep-monthly 2`). Immutability is the cloud copy's job (planned), not the local server's. ## Verifying @@ -134,9 +134,9 @@ Restoring a btrbk snapshot (offline; stop writers first): - **Restore drill**: actually restore a file from a restic snapshot and from a btrbk snapshot, on a schedule. An unverified restore is not a backup. -- **Offsite copy**: youko copies the homelab repositories with `restic copy`; desktops copy their - own. Use an immutable or versioned target (object lock) — that is the layer which survives a - compromised host. +- **Cloud copy**: youko copies the homelab repositories with `restic copy`; desktops copy their own. + The target must not live in the homelab, and should be immutable or versioned (object lock) — that + is the layer which survives a compromised host. - **`restic check` timer**: periodic integrity verification, on the host that holds the password. - **Metrics**: run rest-server with `--prometheus` and chart repository growth in Grafana. - **Recovery key for the homelab password**: `restic-password-homelab.age` is encrypted only to the From a43aa02a6964cf72e726564db47c187c80ac36bd Mon Sep 17 00:00:00 2001 From: Ryan Yin Date: Mon, 21 Sep 2026 20:45:47 +0800 Subject: [PATCH 3/7] docs(secrets): state the recipient invariant A recipient set must answer two questions: who may read it, and how it is recovered once those hosts are gone. Dropping recovery_key while narrowing a secret is easy and silent. --- secrets/README.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/secrets/README.md b/secrets/README.md index 6014b2de..2a530d91 100644 --- a/secrets/README.md +++ b/secrets/README.md @@ -20,6 +20,20 @@ repository. This directory contains this `README.md`, and a `nixos.nix`/`darwin.nix` file that is used to decrypt all my secrets via `agenix`. Then, I can use them in this flake. +## Which Keys Go on a Secret + +Changing a secret's recipients is **two** questions, and both have to be answered: + +- **Who must be able to read it?** Only the hosts that actually need it. Splitting by trust domain + (desktops, servers, one application) keeps a compromised host from reading secrets it has no + business reading. +- **How is it recovered when those hosts are gone?** Every set must keep `recovery_key`. It is held + offline and on no machine, so it does not widen who can read a secret — but without it, the secret + is unrecoverable once the machines that could decrypt it are lost. + +Narrowing a secret's recipients is where this bites: it is easy to drop the offline key along with +the hosts being removed. Re-check both questions after any recipient change. + ## Adding or Updating Secrets > All the operations in this section should be performed in my private repository: `nix-secrets`. From 89e57b5b39d3c83c5b72ebfdc14dbdee85c596a2 Mon Sep 17 00:00:00 2001 From: Ryan Yin Date: Mon, 21 Sep 2026 20:50:11 +0800 Subject: [PATCH 4/7] docs(secrets): state that every secret must be decryptable by the desktops The desktops hold the trusted admin keys and are where secrets are edited and rekeyed from; narrowing a set to the servers alone breaks agenix -r and drops the offline recovery key. --- BACKUP.md | 7 ++++--- secrets/README.md | 16 +++++++--------- 2 files changed, 11 insertions(+), 12 deletions(-) diff --git a/BACKUP.md b/BACKUP.md index eb5b3478..d9879cac 100644 --- a/BACKUP.md +++ b/BACKUP.md @@ -47,6 +47,10 @@ Consequences worth remembering: is the only way to restore its data. - restic's own password is never stored inside a backed-up tree: `etc/agenix` and all key material are excluded (see below). +- Recipients follow the secrets repository's rule: every secret is decryptable by the desktops + (`desktop_keys`, which also carries the offline `recovery_key`), so a desktop can edit or rekey + anything and no secret becomes unrecoverable when a host is lost. The desktop repository password + is the one secret the servers do not get. ## Schedule @@ -139,9 +143,6 @@ Restoring a btrbk snapshot (offline; stop writers first): is the layer which survives a compromised host. - **`restic check` timer**: periodic integrity verification, on the host that holds the password. - **Metrics**: run rest-server with `--prometheus` and chart repository growth in Grafana. -- **Recovery key for the homelab password**: `restic-password-homelab.age` is encrypted only to the - storage hosts' keys, so losing youko's host key makes it unrecoverable. Adding the offline - `recovery_key` to that group fixes it without weakening the desktop/homelab split. ## Components diff --git a/secrets/README.md b/secrets/README.md index 2a530d91..b8a64e33 100644 --- a/secrets/README.md +++ b/secrets/README.md @@ -22,17 +22,15 @@ decrypt all my secrets via `agenix`. Then, I can use them in this flake. ## Which Keys Go on a Secret -Changing a secret's recipients is **two** questions, and both have to be answered: +**Every secret must be decryptable by the desktops.** The desktops hold the trusted admin keys, and +they are where secrets are added, edited and rekeyed — including `agenix -r`, which has to decrypt +every secret in this repository first. Narrowing a secret to the servers alone breaks that. -- **Who must be able to read it?** Only the hosts that actually need it. Splitting by trust domain - (desktops, servers, one application) keeps a compromised host from reading secrets it has no - business reading. -- **How is it recovered when those hosts are gone?** Every set must keep `recovery_key`. It is held - offline and on no machine, so it does not widen who can read a secret — but without it, the secret - is unrecoverable once the machines that could decrypt it are lost. +So every recipient set is `desktop_keys ++ `, and `desktop_keys` carries the +offline `recovery_key`; a lost host therefore never makes a secret unrecoverable. -Narrowing a secret's recipients is where this bites: it is easy to drop the offline key along with -the hosts being removed. Re-check both questions after any recipient change. +The one exception is the desktop's own restic repository password (`restic-password-desktop.age`): +`desktop_keys` alone, because the backup servers must not be able to read desktop data. ## Adding or Updating Secrets From 1cbb52add18de6f078f941fd2dce2f3ce6b5c6e2 Mon Sep 17 00:00:00 2001 From: Ryan Yin Date: Mon, 21 Sep 2026 20:51:13 +0800 Subject: [PATCH 5/7] docs(secrets): desktops and recovery_key have identical access recovery_key is a member of desktop_keys, so every group written as desktop_keys ++ ... covers both. --- secrets/README.md | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/secrets/README.md b/secrets/README.md index b8a64e33..6a1a5abe 100644 --- a/secrets/README.md +++ b/secrets/README.md @@ -22,12 +22,14 @@ decrypt all my secrets via `agenix`. Then, I can use them in this flake. ## Which Keys Go on a Secret -**Every secret must be decryptable by the desktops.** The desktops hold the trusted admin keys, and -they are where secrets are added, edited and rekeyed — including `agenix -r`, which has to decrypt -every secret in this repository first. Narrowing a secret to the servers alone breaks that. +**Every secret is decryptable by the desktops and by the offline `recovery_key`, and those two have +identical access.** The desktops hold the trusted admin keys and are where secrets are added, edited +and rekeyed — including `agenix -r`, which has to decrypt every secret in this repository first. +`recovery_key` is a member of `desktop_keys`, so one rule covers both: every recipient set is +`desktop_keys ++ `. -So every recipient set is `desktop_keys ++ `, and `desktop_keys` carries the -offline `recovery_key`; a lost host therefore never makes a secret unrecoverable. +That also means a lost host never makes a secret unrecoverable, and that narrowing a set to the +servers alone breaks the desktops and the recovery key together. The one exception is the desktop's own restic repository password (`restic-password-desktop.age`): `desktop_keys` alone, because the backup servers must not be able to read desktop data. From 842cd26f64320e2c6a57815856833093511008c1 Mon Sep 17 00:00:00 2001 From: Ryan Yin Date: Mon, 21 Sep 2026 20:56:01 +0800 Subject: [PATCH 6/7] feat(monitoring): scrape the restic rest-server Expose /metrics on the loopback-bound server, scrape it with victoriametrics, and block /metrics on the public vhost. --- hosts/12kingdoms-youko/default.nix | 4 ++++ .../12kingdoms-youko/homelab-services/caddy.nix | 3 +++ .../monitoring/victoriametrics.nix | 16 ++++++++++++++++ 3 files changed, 23 insertions(+) diff --git a/hosts/12kingdoms-youko/default.nix b/hosts/12kingdoms-youko/default.nix index d802b7ea..61971a56 100644 --- a/hosts/12kingdoms-youko/default.nix +++ b/hosts/12kingdoms-youko/default.nix @@ -73,6 +73,10 @@ in dataDir = "/data/backups/rest-server"; privateRepos = true; "htpasswd-file" = config.age.secrets."restic-rest-htpasswd".path; + # expose /metrics for victoriametrics; unauthenticated is fine because the + # server only listens on loopback + prometheus = true; + extraFlags = [ "--prometheus-no-auth" ]; }; # repositories live on the HDD, which has to be mounted first diff --git a/hosts/12kingdoms-youko/homelab-services/caddy.nix b/hosts/12kingdoms-youko/homelab-services/caddy.nix index e9cb8c22..1b500dfb 100644 --- a/hosts/12kingdoms-youko/homelab-services/caddy.nix +++ b/hosts/12kingdoms-youko/homelab-services/caddy.nix @@ -66,6 +66,9 @@ in # the restic REST server the backup clients push to virtualHosts."restic.writefor.fun".extraConfig = '' ${hostCommonConfig} + # metrics are scraped over loopback only, never through this vhost + @metrics path /metrics + respond @metrics 403 reverse_proxy http://localhost:8000 ''; # Every other name is not a service on this host: answer 404 instead of diff --git a/hosts/12kingdoms-youko/homelab-services/monitoring/victoriametrics.nix b/hosts/12kingdoms-youko/homelab-services/monitoring/victoriametrics.nix index 1201c2d5..5bfb747a 100644 --- a/hosts/12kingdoms-youko/homelab-services/monitoring/victoriametrics.nix +++ b/hosts/12kingdoms-youko/homelab-services/monitoring/victoriametrics.nix @@ -77,6 +77,22 @@ in } */ + { + job_name = "restic-rest-server"; + scrape_interval = "60s"; + metrics_path = "/metrics"; + static_configs = [ + { + # same-host rest-server bound to loopback (127.0.0.1:8000) + targets = [ "127.0.0.1:8000" ]; + labels.type = "app"; + labels.app = "restic"; + labels.host = "youko"; + labels.env = "homelab"; + labels.cluster = "homelab"; + } + ]; + } { job_name = "v2ray-exporter"; scrape_interval = "30s"; From 37f29154d1b7b908ca517b401a81adc22beb7206 Mon Sep 17 00:00:00 2001 From: Ryan Yin Date: Mon, 21 Sep 2026 20:57:41 +0800 Subject: [PATCH 7/7] docs(backup): record what the metrics actually expose rest-server only ships the go/promhttp defaults, so repository growth is charted from node-exporter's filesystem metrics instead. --- BACKUP.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/BACKUP.md b/BACKUP.md index d9879cac..fbe490bf 100644 --- a/BACKUP.md +++ b/BACKUP.md @@ -130,6 +130,10 @@ Restoring a btrbk snapshot (offline; stop writers first): - `restic-rest-server`'s log must not print `Invalid htpasswd entry`: that means the client's REST password and the server's htpasswd disagree, and the credentials pair needs rebuilding from one password. +- Metrics: the rest-server exposes the Go/promhttp defaults on loopback (scraped as the + `restic-rest-server` job; `/metrics` is blocked on the public vhost). Repository growth is charted + from `node-exporter`'s filesystem metrics for youko's `/data` mount instead — rest-server itself + has no per-repository size metrics. - Failures are not silent: a failed `restic-backups-homelab` starts `restic-backup-notify`, which posts to alertmanager and, above info severity, on to telegram. alertmanager clears it after `resolve_timeout`. @@ -142,7 +146,6 @@ Restoring a btrbk snapshot (offline; stop writers first): The target must not live in the homelab, and should be immutable or versioned (object lock) — that is the layer which survives a compromised host. - **`restic check` timer**: periodic integrity verification, on the host that holds the password. -- **Metrics**: run rest-server with `--prometheus` and chart repository growth in Grafana. ## Components