feat(restic): move idols-ai's off-host backup to the REST server

Split the repository password into desktop and homelab groups, add the
environmentFile/pruneOpts module options, and point the desktop at youko's
REST endpoint.

flake.lock

@@ -503,10 +503,10 @@
     "mysecrets": {
       "flake": false,
       "locked": {
-        "lastModified": 1789987762,
-        "narHash": "sha256-gfrl1P3WdGBMdOZ9gaGg1UQwjGGwPzcLQUjUEhmBJOE=",
+        "lastModified": 1789993347,
+        "narHash": "sha256-6B1ho/wSfgOuUPLBnthbpjz0XkEdxV45RkRsr6QMJ40=",
         "ref": "refs/heads/main",
-        "rev": "00f4f7fdf580eb1807e742977fd581dde137e575",
+        "rev": "dd48c84595ad4fab4b420f79e8458f03582129e8",
         "shallow": true,
         "type": "git",
         "url": "ssh://git@github.com/ryan4yin/nix-secrets.git"

hosts/idols-ai/default.nix

@@ -34,6 +34,8 @@ in
     ./preservation.nix
     ./secureboot.nix
 
+    ./restic.nix
+
     # others
     ./ai
   ];

hosts/idols-ai/restic.nix

@@ -1,40 +1,90 @@
 {
+  config,
   ...
 }:
 {
   # Part 1 of the backup plan: a local restic repository on this machine.
   # The off-machine target (youko, or a cloud) is a later step.
   #
-  # Be specific here and keep `exclude` minimal: an allowlist of the few
-  # directories that are actually irreplaceable beats backing up a broad tree
-  # and excluding half of it. Project checkouts under ~/codes are tracked in
-  # git (or have remotes), so they are not backed up at all.
+  # Be specific here and keep `exclude` minimal: an allowlist of the
+  # irreplaceable directories beats backing up a broad tree and excluding half
+  # of it. Large files anywhere are skipped by size (excludeLargerThan), which
+  # catches re-downloadable artifacts (models, datasets, archives) without
+  # guessing file types or project names.
+  # Back up to the homelab backup server (youko) over its restic REST server.
+  # A repository on this machine would die with it. The repository password is
+  # desktop-only (the server holds the homelab one, not this); the REST
+  # credentials come from a systemd EnvironmentFile so they stay out of the URL.
   modules.restic-backup = {
     enable = true;
-    repository = "/var/lib/backups/restic/idols-ai";
+    repository = "rest:https://backup.writefor.fun/idols-ai/";
+    environmentFile = config.age.secrets."restic-rest-credentials".path;
 
     paths = [
       "/home/ryan/Documents"
       "/home/ryan/work"
       "/home/ryan/Pictures"
+      "/home/ryan/codes"
       "/home/ryan/nix-config"
       "/home/ryan/.local/state"
     ];
 
+    # anything bigger than this is a re-downloadable artifact (models, datasets,
+    # archives); a size cap beats guessing file types
+    excludeLargerThan = "500M";
+
+    # The server is append-only (it rejects deletion), so the client cannot
+    # forget or prune: retention has to be applied by hand, by running
+    # `restic forget --prune` while the server is temporarily not in
+    # append-only mode. Snapshots therefore accumulate here; deduplication
+    # keeps that cheap.
+    pruneOpts = [ ];
+
     exclude = [
-      # generic patterns only; this repository is public
+      # Generic patterns only; this repository is public.
+      #
+      # node / JS
       "**/node_modules"
-      "**/target"
-      "**/.venv"
-      "**/venv"
       "**/.next"
       "**/dist"
+      # rust
+      "**/target"
+      # c / c++
+      "**/CMakeFiles"
+      "**/cmake-build-*"
+      "**/compile_commands.json"
+      # python
+      "**/__pycache__"
+      "**/*.egg-info"
+      "**/.venv"
+      "**/venv"
+      "**/site-packages"
+      "**/.pytest_cache"
+      "**/.mypy_cache"
+      "**/.ruff_cache"
+      "**/.tox"
+      "**/.nox"
+      # go
+      "**/vendor"
+      # generic build dirs and tooling
       "**/build"
       "**/.direnv"
       "**/result*"
-      "**/__pycache__"
-      "**/.pytest_cache"
-      "**/.ruff_cache"
+      "**/.terraform"
+      # model weights in the common formats, and language caches
+      "**/*.gguf"
+      "**/*.ggml"
+      "**/*.safetensors"
+      "**/*.pth"
+      "**/*.pt"
+      "**/*.ckpt"
+      "**/*.onnx"
+      "**/*.h5"
+      "**/*.keras"
+      "**/*.tflite"
+      "**/.cache"
+      # git object packs: re-fetchable from the remotes
+      "**/.git/objects/pack"
     ];
   };
 }

modules/nixos/base/restic-backup.nix

@@ -68,6 +68,44 @@ in
         not-yet-mounted directory.
       '';
     };
+
+    excludeLargerThan = lib.mkOption {
+      type = lib.types.nullOr lib.types.str;
+      default = null;
+      example = "500M";
+      description = ''
+        Skip any file larger than this (restic's `--exclude-larger-than`). A size
+        cap is more reliable than guessing file types when the tree contains
+        large re-downloadable artifacts.
+      '';
+    };
+
+    environmentFile = lib.mkOption {
+      type = lib.types.nullOr lib.types.path;
+      default = null;
+      example = config.age.secrets."restic-rest-credentials".path;
+      description = ''
+        systemd `EnvironmentFile` with restic environment variables. Needed for
+        a `rest:` repository, whose credentials are environment-only
+        (`RESTIC_REST_USERNAME` / `RESTIC_REST_PASSWORD`) and must not appear in
+        the repository URL.
+      '';
+    };
+
+    pruneOpts = lib.mkOption {
+      type = lib.types.listOf lib.types.str;
+      default = [
+        "--keep-daily 3"
+        "--keep-weekly 2"
+        "--keep-monthly 2"
+      ];
+      description = ''
+        Retention passed to `restic forget --prune`. Leave it empty when the
+        repository is an append-only server: that server rejects deletion, so
+        the client cannot forget or prune (the backup would otherwise fail
+        after uploading).
+      '';
+    };
   };
 
   config = lib.mkIf cfg.enable {
@@ -82,20 +120,12 @@ in
       # Relative to the backed-up tree, so they work for both the snapshot and
       # a direct path.
       exclude = [
-        # regenerable, huge, or unsuitable for file-level backup
+        # Universal: keys and credentials are never backed up, on any host.
+        # (restic's own password lives in /etc/agenix, so backing that up would
+        # store the repository's password inside the repository.)
         #
-        # the whole podman storage tree: the image layers are re-pullable, and
-        # its storage DB is not consistent when copied from a running podman.
-        # (uptime-kuma uses a named volume under here; its data is not wanted.)
-        "var/lib/containers"
-        "var/lib/microvms"
-        "var/lib/libvirt"
-        "nfs"
-        "var/cache"
-        "var/tmp"
-        "var/log"
-        "*.qcow2"
-        # keys and credentials are never backed up
+        # Host-specific excludes (regenerable bulk, VM images, ...) belong in
+        # the host's own config.
         "etc/agenix"
         "etc/ssh/ssh_host_*"
         "**/.ssh"
@@ -110,12 +140,19 @@ in
         RandomizedDelaySec = "1h";
       };
 
-      pruneOpts = [
-        "--keep-daily 3"
-        "--keep-weekly 2"
-        "--keep-monthly 2"
+      inherit (cfg) pruneOpts;
+    }
+    // lib.optionalAttrs (cfg.excludeLargerThan != null) {
+      extraBackupArgs = [
+        "--exclude-larger-than"
+        cfg.excludeLargerThan
       ];
     }
+
+    // lib.optionalAttrs (cfg.environmentFile != null) {
+      inherit (cfg) environmentFile;
+    }
+
     // lib.optionalAttrs (cfg.snapshotSource != null) {
       backupPrepareCommand = ''
         ${pkgs.btrfs-progs}/bin/btrfs subvolume snapshot -r ${cfg.snapshotSource} ${snapshotPath}

secrets/nixos.nix

@@ -213,14 +213,38 @@ in
       };
     })
 
-    (mkIf (cfg.desktop.enable || cfg.server.storage.enable) {
-      # the restic repository password, needed by modules.restic-backup on both
-      # the homelab servers and the desktops
+    (mkIf cfg.desktop.enable {
+      # Desktops use their own restic repository password, NOT the homelab's:
+      # the backup server holds the homelab one, so a shared password would let
+      # that server decrypt desktop data.
       age.secrets."restic-password" = {
-        file = "${mysecrets}/restic-password.age";
+        file = "${mysecrets}/restic-password-desktop.age";
         mode = "0400";
         owner = "root";
       };
+      # credentials for the restic REST server that receives this backup
+      age.secrets."restic-rest-credentials" = {
+        file = "${mysecrets}/restic-rest-credentials.age";
+        mode = "0400";
+        owner = "root";
+      };
+    })
+
+    (mkIf cfg.server.storage.enable {
+      # the homelab's restic repository password, shared by the homelab hosts
+      # (this host holds it, so it can copy/verify homelab repositories)
+      age.secrets."restic-password" = {
+        file = "${mysecrets}/restic-password-homelab.age";
+        mode = "0400";
+        owner = "root";
+      };
+      # htpasswd for the restic REST server this host runs; the service runs as
+      # the unprivileged `restic` user, which must be able to read it
+      age.secrets."restic-rest-htpasswd" = {
+        file = "${mysecrets}/restic-rest-htpasswd.age";
+        mode = "0400";
+        owner = "restic";
+      };
     })
 
     (mkIf cfg.server.storage.enable {