diff --git a/flake.lock b/flake.lock index edd7d0a7..8ef92aa0 100644 --- a/flake.lock +++ b/flake.lock @@ -503,10 +503,10 @@ "mysecrets": { "flake": false, "locked": { - "lastModified": 1789987762, - "narHash": "sha256-gfrl1P3WdGBMdOZ9gaGg1UQwjGGwPzcLQUjUEhmBJOE=", + "lastModified": 1789993347, + "narHash": "sha256-6B1ho/wSfgOuUPLBnthbpjz0XkEdxV45RkRsr6QMJ40=", "ref": "refs/heads/main", - "rev": "00f4f7fdf580eb1807e742977fd581dde137e575", + "rev": "dd48c84595ad4fab4b420f79e8458f03582129e8", "shallow": true, "type": "git", "url": "ssh://git@github.com/ryan4yin/nix-secrets.git" diff --git a/hosts/idols-ai/default.nix b/hosts/idols-ai/default.nix index 9a176ba4..3f5aff78 100644 --- a/hosts/idols-ai/default.nix +++ b/hosts/idols-ai/default.nix @@ -34,6 +34,8 @@ in ./preservation.nix ./secureboot.nix + ./restic.nix + # others ./ai ]; diff --git a/hosts/idols-ai/restic.nix b/hosts/idols-ai/restic.nix index 87b807aa..b1587097 100644 --- a/hosts/idols-ai/restic.nix +++ b/hosts/idols-ai/restic.nix @@ -1,40 +1,90 @@ { + config, ... }: { # Part 1 of the backup plan: a local restic repository on this machine. # The off-machine target (youko, or a cloud) is a later step. # - # Be specific here and keep `exclude` minimal: an allowlist of the few - # directories that are actually irreplaceable beats backing up a broad tree - # and excluding half of it. Project checkouts under ~/codes are tracked in - # git (or have remotes), so they are not backed up at all. + # Be specific here and keep `exclude` minimal: an allowlist of the + # irreplaceable directories beats backing up a broad tree and excluding half + # of it. Large files anywhere are skipped by size (excludeLargerThan), which + # catches re-downloadable artifacts (models, datasets, archives) without + # guessing file types or project names. + # Back up to the homelab backup server (youko) over its restic REST server. + # A repository on this machine would die with it. The repository password is + # desktop-only (the server holds the homelab one, not this); the REST + # credentials come from a systemd EnvironmentFile so they stay out of the URL. modules.restic-backup = { enable = true; - repository = "/var/lib/backups/restic/idols-ai"; + repository = "rest:https://backup.writefor.fun/idols-ai/"; + environmentFile = config.age.secrets."restic-rest-credentials".path; paths = [ "/home/ryan/Documents" "/home/ryan/work" "/home/ryan/Pictures" + "/home/ryan/codes" "/home/ryan/nix-config" "/home/ryan/.local/state" ]; + # anything bigger than this is a re-downloadable artifact (models, datasets, + # archives); a size cap beats guessing file types + excludeLargerThan = "500M"; + + # The server is append-only (it rejects deletion), so the client cannot + # forget or prune: retention has to be applied by hand, by running + # `restic forget --prune` while the server is temporarily not in + # append-only mode. Snapshots therefore accumulate here; deduplication + # keeps that cheap. + pruneOpts = [ ]; + exclude = [ - # generic patterns only; this repository is public + # Generic patterns only; this repository is public. + # + # node / JS "**/node_modules" - "**/target" - "**/.venv" - "**/venv" "**/.next" "**/dist" + # rust + "**/target" + # c / c++ + "**/CMakeFiles" + "**/cmake-build-*" + "**/compile_commands.json" + # python + "**/__pycache__" + "**/*.egg-info" + "**/.venv" + "**/venv" + "**/site-packages" + "**/.pytest_cache" + "**/.mypy_cache" + "**/.ruff_cache" + "**/.tox" + "**/.nox" + # go + "**/vendor" + # generic build dirs and tooling "**/build" "**/.direnv" "**/result*" - "**/__pycache__" - "**/.pytest_cache" - "**/.ruff_cache" + "**/.terraform" + # model weights in the common formats, and language caches + "**/*.gguf" + "**/*.ggml" + "**/*.safetensors" + "**/*.pth" + "**/*.pt" + "**/*.ckpt" + "**/*.onnx" + "**/*.h5" + "**/*.keras" + "**/*.tflite" + "**/.cache" + # git object packs: re-fetchable from the remotes + "**/.git/objects/pack" ]; }; } diff --git a/modules/nixos/base/restic-backup.nix b/modules/nixos/base/restic-backup.nix index 55c58287..96f8d641 100644 --- a/modules/nixos/base/restic-backup.nix +++ b/modules/nixos/base/restic-backup.nix @@ -68,6 +68,44 @@ in not-yet-mounted directory. ''; }; + + excludeLargerThan = lib.mkOption { + type = lib.types.nullOr lib.types.str; + default = null; + example = "500M"; + description = '' + Skip any file larger than this (restic's `--exclude-larger-than`). A size + cap is more reliable than guessing file types when the tree contains + large re-downloadable artifacts. + ''; + }; + + environmentFile = lib.mkOption { + type = lib.types.nullOr lib.types.path; + default = null; + example = config.age.secrets."restic-rest-credentials".path; + description = '' + systemd `EnvironmentFile` with restic environment variables. Needed for + a `rest:` repository, whose credentials are environment-only + (`RESTIC_REST_USERNAME` / `RESTIC_REST_PASSWORD`) and must not appear in + the repository URL. + ''; + }; + + pruneOpts = lib.mkOption { + type = lib.types.listOf lib.types.str; + default = [ + "--keep-daily 3" + "--keep-weekly 2" + "--keep-monthly 2" + ]; + description = '' + Retention passed to `restic forget --prune`. Leave it empty when the + repository is an append-only server: that server rejects deletion, so + the client cannot forget or prune (the backup would otherwise fail + after uploading). + ''; + }; }; config = lib.mkIf cfg.enable { @@ -82,20 +120,12 @@ in # Relative to the backed-up tree, so they work for both the snapshot and # a direct path. exclude = [ - # regenerable, huge, or unsuitable for file-level backup + # Universal: keys and credentials are never backed up, on any host. + # (restic's own password lives in /etc/agenix, so backing that up would + # store the repository's password inside the repository.) # - # the whole podman storage tree: the image layers are re-pullable, and - # its storage DB is not consistent when copied from a running podman. - # (uptime-kuma uses a named volume under here; its data is not wanted.) - "var/lib/containers" - "var/lib/microvms" - "var/lib/libvirt" - "nfs" - "var/cache" - "var/tmp" - "var/log" - "*.qcow2" - # keys and credentials are never backed up + # Host-specific excludes (regenerable bulk, VM images, ...) belong in + # the host's own config. "etc/agenix" "etc/ssh/ssh_host_*" "**/.ssh" @@ -110,12 +140,19 @@ in RandomizedDelaySec = "1h"; }; - pruneOpts = [ - "--keep-daily 3" - "--keep-weekly 2" - "--keep-monthly 2" + inherit (cfg) pruneOpts; + } + // lib.optionalAttrs (cfg.excludeLargerThan != null) { + extraBackupArgs = [ + "--exclude-larger-than" + cfg.excludeLargerThan ]; } + + // lib.optionalAttrs (cfg.environmentFile != null) { + inherit (cfg) environmentFile; + } + // lib.optionalAttrs (cfg.snapshotSource != null) { backupPrepareCommand = '' ${pkgs.btrfs-progs}/bin/btrfs subvolume snapshot -r ${cfg.snapshotSource} ${snapshotPath} diff --git a/secrets/nixos.nix b/secrets/nixos.nix index b159d8e7..e5b42ee2 100644 --- a/secrets/nixos.nix +++ b/secrets/nixos.nix @@ -213,14 +213,38 @@ in }; }) - (mkIf (cfg.desktop.enable || cfg.server.storage.enable) { - # the restic repository password, needed by modules.restic-backup on both - # the homelab servers and the desktops + (mkIf cfg.desktop.enable { + # Desktops use their own restic repository password, NOT the homelab's: + # the backup server holds the homelab one, so a shared password would let + # that server decrypt desktop data. age.secrets."restic-password" = { - file = "${mysecrets}/restic-password.age"; + file = "${mysecrets}/restic-password-desktop.age"; mode = "0400"; owner = "root"; }; + # credentials for the restic REST server that receives this backup + age.secrets."restic-rest-credentials" = { + file = "${mysecrets}/restic-rest-credentials.age"; + mode = "0400"; + owner = "root"; + }; + }) + + (mkIf cfg.server.storage.enable { + # the homelab's restic repository password, shared by the homelab hosts + # (this host holds it, so it can copy/verify homelab repositories) + age.secrets."restic-password" = { + file = "${mysecrets}/restic-password-homelab.age"; + mode = "0400"; + owner = "root"; + }; + # htpasswd for the restic REST server this host runs; the service runs as + # the unprivileged `restic` user, which must be able to read it + age.secrets."restic-rest-htpasswd" = { + file = "${mysecrets}/restic-rest-htpasswd.age"; + mode = "0400"; + owner = "restic"; + }; }) (mkIf cfg.server.storage.enable {