diff --git a/flake.lock b/flake.lock index 171f32ed..edd7d0a7 100644 --- a/flake.lock +++ b/flake.lock @@ -503,10 +503,10 @@ "mysecrets": { "flake": false, "locked": { - "lastModified": 1789975872, - "narHash": "sha256-DQrX1au7BWEzXJqR2FBiSYdF0DWd5NMh12y6ovtn41o=", + "lastModified": 1789987762, + "narHash": "sha256-gfrl1P3WdGBMdOZ9gaGg1UQwjGGwPzcLQUjUEhmBJOE=", "ref": "refs/heads/main", - "rev": "4def15deec5c0538df16ad2d49028298ee81d7ab", + "rev": "00f4f7fdf580eb1807e742977fd581dde137e575", "shallow": true, "type": "git", "url": "ssh://git@github.com/ryan4yin/nix-secrets.git" diff --git a/hosts/12kingdoms-youko/default.nix b/hosts/12kingdoms-youko/default.nix index c25fa1da..0160579f 100644 --- a/hosts/12kingdoms-youko/default.nix +++ b/hosts/12kingdoms-youko/default.nix @@ -28,6 +28,16 @@ in coreModule ]; + # Back up the preserved tree through a short-lived read-only snapshot, so the + # backup is a consistent point-in-time; the module's excludes drop the + # regenerable bulk and all keys/credentials. + modules.restic-backup = { + enable = true; + repository = "/data/backups/restic/youko"; + snapshotSource = "/btr_pool/@persistent"; + requiresMountsFor = "/data/backups"; + }; + modules.btrbk.enable = true; boot.kernelParams = [ diff --git a/hosts/12kingdoms-youko/homelab-services/restic.nix b/hosts/12kingdoms-youko/homelab-services/restic.nix deleted file mode 100644 index 00e5d0f5..00000000 --- a/hosts/12kingdoms-youko/homelab-services/restic.nix +++ /dev/null @@ -1,87 +0,0 @@ -{ pkgs, ... }: -let - passwordFile = "/etc/agenix/restic-password"; - sshKeyPath = "/etc/agenix/ssh-key-for-restic-backup"; - rcloneConfigFile = "/etc/agenix/rclone-conf-for-restic-backup"; -in -{ - # DISABLED: this backup has never worked. It needs three agenix secrets that - # do not exist in nix-secrets yet (`restic-password`, `ssh-key-for-restic-backup`, - # and an rclone config with the `smb-downloads` remote; the existing - # `rclone.conf` secret is only defined for the desktop secret set). Re-enable - # once the secrets are provisioned. - # - # https://github.com/NixOS/nixpkgs/blob/nixos-26.05/nixos/modules/services/backup/restic.nix - /* - services.restic.backups = { - homelab-backup = { - inherit passwordFile; - initialize = true; # Initialize the repository if it doesn't exist. - repository = "rclone:smb-downloads:/Downloads/kubevirt-backup/"; # backup to a rclone remote - - # rclone related - # rcloneOptions = { - # bwlimit = "100M"; # Limit the bandwidth used by rclone. - # }; - inherit rcloneConfigFile; - - # Which local paths to backup, in addition to ones specified via `dynamicFilesFrom`. - paths = [ - "/tmp/restic-backup-temp" - ]; - # - # A script that produces a list of files to back up. The - # results of this command are given to the '--files-from' - # option. The result is merged with paths specified via `paths`. - # dynamicFilesFrom = "find /home/matt/git -type d -name .git"; - # - # Patterns to exclude when backing up. See - # https://restic.readthedocs.io/en/latest/040_backup.html#excluding-files - # for details on syntax. - exclude = [ ]; - - # A script that must run before starting the backup process. - backupPrepareCommand = '' - ${pkgs.nushell}/bin/nu -c ' - let kubevirt_nodes = [ - "shoryu" - "shushou" - "youko" - ] - - kubevirt_nodes | each {|it| - ^${pkgs.rsync}/bin/rsync -avz -e $"ssh -i ${sshKeyPath}" $"($it):/persistent/" $"/tmp/restic-backup-temp/($it)" - } - ' - ''; - # A script that must run after finishing the backup process. - backupCleanupCommand = "rm -rf /tmp/restic-backup-temp"; - - # Extra extended options to be passed to the restic --option flag. - # extraOptions = []; - - # Extra arguments passed to restic backup. - # extraBackupArgs = [ - # "--exclude-file=/etc/restic/excludes-list" - # ]; - - # repository = "/mnt/backup-hdd"; # backup to a local directory - # When to run the backup. See {manpage}`systemd.timer(5)` for details. - timerConfig = { - OnCalendar = "01:30"; - RandomizedDelaySec = "1h"; - }; - # A list of options (--keep-* et al.) for 'restic forget --prune', - # to automatically prune old snapshots. - # The 'forget' command is run *after* the 'backup' command, so - # keep that in mind when constructing the --keep-* options. - pruneOpts = [ - "--keep-daily 3" - "--keep-weekly 3" - "--keep-monthly 3" - "--keep-yearly 3" - ]; - }; - }; - */ -} diff --git a/modules/nixos/base/btrbk.nix b/modules/nixos/base/btrbk.nix index 4fd7f839..c263ca52 100644 --- a/modules/nixos/base/btrbk.nix +++ b/modules/nixos/base/btrbk.nix @@ -19,8 +19,12 @@ in # /btr_pool/@snapshots/@persistent. # # These are same-filesystem snapshots: they protect against accidental - # deletion and bad edits, NOT against disk loss. Set `target` (plus - # `services.btrbk.sshAccess` on the receiving host) for off-host backups. + # deletion and bad edits, NOT against disk loss. Off-host copies are handled + # by restic (lib/genResticBackup.nix), which can exclude the regenerable bulk + # at the file level; btrbk works at the subvolume level, so a `target` would + # also ship podman's overlay storage and the VM disk images. Set `target` + # (plus `services.btrbk.sshAccess` on the receiving host) only if you want + # that. # # The host MUST mount the btrfs top-level subvolume (subvolid=5) at `volume`; # this is enforced by an assertion so a missing mount fails evaluation instead diff --git a/modules/nixos/base/restic-backup.nix b/modules/nixos/base/restic-backup.nix new file mode 100644 index 00000000..55c58287 --- /dev/null +++ b/modules/nixos/base/restic-backup.nix @@ -0,0 +1,134 @@ +{ + config, + lib, + pkgs, + ... +}: +let + cfg = config.modules.restic-backup; + + # A short-lived, read-only btrfs snapshot taken by the backup itself, so the + # backup is a consistent point-in-time (a live copy of, say, a running + # postgres data directory is only crash-consistent). It is deleted right + # after, so it is not a second backup - just a consistency step. + snapshotPath = "${builtins.dirOf cfg.snapshotSource}/@restic-snapshot"; +in +{ + # ================================================================== + # + # A restic backup to a LOCAL repository (part 1 of the backup plan). + # + # Part 2 (copying the critical snapshots to a cloud object store with + # `restic copy`) is a later step. btrbk keeps making the cheap local btrfs + # snapshots for rollbacks; off-host copies are restic's job (restic can + # exclude the regenerable bulk at the file level). + # + # Keys and credentials are NEVER included: restic's own password lives in + # /etc/agenix, so backing that up would store the repository's password + # inside the repository. + # + # ================================================================== + options.modules.restic-backup = { + enable = lib.mkEnableOption "a restic backup (local repository)"; + + repository = lib.mkOption { + type = lib.types.str; + example = "/data/backups/restic/youko"; + description = "Restic repository, e.g. a path under a mounted backup disk."; + }; + + snapshotSource = lib.mkOption { + type = lib.types.nullOr lib.types.str; + default = null; + example = "/btr_pool/@persistent"; + description = '' + btrfs subvolume to back up through a short-lived read-only snapshot. + When null, `paths` is backed up directly. + ''; + }; + + paths = lib.mkOption { + type = lib.types.listOf lib.types.str; + default = [ ]; + description = "Paths to back up (used when {option}`snapshotSource` is null)."; + }; + + exclude = lib.mkOption { + type = lib.types.listOf lib.types.str; + default = [ ]; + description = "Extra exclude patterns, on top of the built-in ones."; + }; + + requiresMountsFor = lib.mkOption { + type = lib.types.nullOr lib.types.str; + default = null; + description = '' + Mount point the repository lives on, when it is a `nofail` mount: the + backup is then ordered after that mount instead of running against a + not-yet-mounted directory. + ''; + }; + }; + + config = lib.mkIf cfg.enable { + services.restic.backups.homelab = { + inherit (cfg) repository; + initialize = true; + # take the path from agenix itself instead of hardcoding its secretsDir + passwordFile = config.age.secrets."restic-password".path; + + paths = if cfg.snapshotSource != null then [ snapshotPath ] else cfg.paths; + + # Relative to the backed-up tree, so they work for both the snapshot and + # a direct path. + exclude = [ + # regenerable, huge, or unsuitable for file-level backup + # + # the whole podman storage tree: the image layers are re-pullable, and + # its storage DB is not consistent when copied from a running podman. + # (uptime-kuma uses a named volume under here; its data is not wanted.) + "var/lib/containers" + "var/lib/microvms" + "var/lib/libvirt" + "nfs" + "var/cache" + "var/tmp" + "var/log" + "*.qcow2" + # keys and credentials are never backed up + "etc/agenix" + "etc/ssh/ssh_host_*" + "**/.ssh" + "**/.gnupg" + "**/.aws" + "**/.config/gcloud" + ] + ++ cfg.exclude; + + timerConfig = { + OnCalendar = "01:30"; + RandomizedDelaySec = "1h"; + }; + + pruneOpts = [ + "--keep-daily 3" + "--keep-weekly 2" + "--keep-monthly 2" + ]; + } + // lib.optionalAttrs (cfg.snapshotSource != null) { + backupPrepareCommand = '' + ${pkgs.btrfs-progs}/bin/btrfs subvolume snapshot -r ${cfg.snapshotSource} ${snapshotPath} + ''; + backupCleanupCommand = '' + ${pkgs.btrfs-progs}/bin/btrfs subvolume delete ${snapshotPath} + ''; + }; + + systemd.services.restic-backups-homelab.unitConfig = + lib.optionalAttrs (cfg.requiresMountsFor != null) + { + RequiresMountsFor = cfg.requiresMountsFor; + }; + }; +} diff --git a/secrets/nixos.nix b/secrets/nixos.nix index c454ebf2..63e10cd2 100644 --- a/secrets/nixos.nix +++ b/secrets/nixos.nix @@ -220,6 +220,12 @@ in mode = "0400"; owner = "root"; }; + # restic repository password (used by modules.restic-backup) + "restic-password" = { + file = "${mysecrets}/restic-password.age"; + mode = "0400"; + owner = "root"; + }; }; # place secrets in /etc/