feat(backup): notify on failure, and 404 unconfigured hostnames

A failed backup was silent. Post to alertmanager (which routes above-info
to telegram), and have caddy answer 404 instead of its empty 200 for
*.writefor.fun names that are not services.

BACKUP.md

@@ -126,6 +126,22 @@ Restoring a btrbk snapshot (offline; stop writers first):
 - `restic-rest-server`'s log must not print `Invalid htpasswd entry`: that means the client's REST
   password and the server's htpasswd disagree, and the credentials pair needs rebuilding from one
   password.
+- Failures are not silent: a failed `restic-backups-homelab` starts `restic-backup-notify`, which
+  posts to alertmanager and, above info severity, on to telegram. alertmanager clears it after
+  `resolve_timeout`.
+
+## Planned
+
+- **Restore drill**: actually restore a file from a restic snapshot and from a btrbk snapshot, on a
+  schedule. An unverified restore is not a backup.
+- **Offsite copy**: youko copies the homelab repositories with `restic copy`; desktops copy their
+  own. Use an immutable or versioned target (object lock) — that is the layer which survives a
+  compromised host.
+- **`restic check` timer**: periodic integrity verification, on the host that holds the password.
+- **Metrics**: run rest-server with `--prometheus` and chart repository growth in Grafana.
+- **Recovery key for the homelab password**: `restic-password-homelab.age` is encrypted only to the
+  storage hosts' keys, so losing youko's host key makes it unrecoverable. Adding the offline
+  `recovery_key` to that group fixes it without weakening the desktop/homelab split.
 
 ## Components
 

hosts/12kingdoms-youko/homelab-services/caddy.nix

@@ -68,6 +68,12 @@ in
       ${hostCommonConfig}
       reverse_proxy http://localhost:8000
     '';
+    # Every other name is not a service on this host: answer 404 instead of
+    # caddy's empty 200 fallback, which hides typos.
+    virtualHosts."*.writefor.fun".extraConfig = ''
+      ${hostCommonConfig}
+      respond 404
+    '';
     virtualHosts."transmission.writefor.fun".extraConfig = ''
       ${hostCommonConfig}
       encode zstd gzip

modules/nixos/base/restic-backup.nix

@@ -109,6 +109,16 @@ in
       description = "Extra random delay on top of {option}`onCalendar`, to smooth load.";
     };
 
+    alertManagerUrl = lib.mkOption {
+      type = lib.types.str;
+      default = "https://alertmanager.writefor.fun";
+      description = ''
+        Alertmanager to POST a failure alert to (`/api/v2/alerts`). youko routes
+        anything above info severity to telegram, so a failed backup is not
+        silent.
+      '';
+    };
+
     pruneOpts = lib.mkOption {
       type = lib.types.listOf lib.types.str;
       default = [
@@ -183,5 +193,22 @@ in
         {
           RequiresMountsFor = cfg.requiresMountsFor;
         };
+
+    # A failed backup must not be silent: report it to alertmanager, which
+    # routes anything above info severity to telegram.
+    systemd.services.restic-backups-homelab.onFailure = [ "restic-backup-notify.service" ];
+
+    systemd.services.restic-backup-notify = {
+      description = "Report a failed restic backup";
+      serviceConfig.Type = "oneshot";
+      script = ''
+        ${pkgs.curl}/bin/curl -sS -o /dev/null -X POST ${cfg.alertManagerUrl}/api/v2/alerts \
+          -H 'Content-Type: application/json' \
+          --data "[{ \
+            \"labels\": { \"alertname\": \"ResticBackupFailed\", \"severity\": \"critical\", \"host\": \"${config.networking.hostName}\" }, \
+            \"annotations\": { \"summary\": \"restic backup failed on ${config.networking.hostName}\", \"description\": \"journalctl -u restic-backups-homelab\" } \
+          }]"
+      '';
+    };
   };
 }