From f8d12ada24b87969d18515c2e78b4769eb587c0d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?ryan4yin=20=7C=20=E4=BA=8C=E8=8A=B1?= Date: Sat, 19 Sep 2026 11:43:11 +0800 Subject: [PATCH] feat(btrbk): make snapshots explicit, guarded, and stored in @snapshots (#271) * feat(btrbk): make snapshots explicit and guarded Replace the always-on, hardcoded module with modules.btrbk options (enable/volume/subvolume/snapshotDir/target/onCalendar), and assert the btrfs top-level subvolume is mounted so a missing mount fails evaluation instead of silently failing at runtime. Store snapshots in the @snapshots subvolume (snapshot_dir=@snapshots) rather than next to the live subvolume, make the off-host target explicit and off by default, and document that local snapshots are not off-host backups. Enable the feature explicitly on idols-ai, shoukei and the three kubevirt hosts; qemu-guest now disables it through the option. Add eval tests for shoukei and ai. * feat(monitoring): alert on btrbk snapshot failure Add HostBtrbkSnapshotFailed on node_systemd_unit_state for btrbk-btrbk.service, so a broken snapshot schedule alerts instead of only surfacing in the journal. --- hosts/12kingdoms-shoukei/default.nix | 8 ++ hosts/idols-ai/default.nix | 2 + .../monitoring/alert_rules/node-exporter.yml | 11 ++ hosts/k8s/kubevirt-shoryu/default.nix | 2 + hosts/k8s/kubevirt-shushou/default.nix | 2 + hosts/k8s/kubevirt-youko/default.nix | 2 + modules/nixos/base/btrbk.nix | 120 +++++++++++++----- modules/nixos/server/qemu-guest.nix | 2 +- .../aarch64-linux/tests/btrbk/expected.nix | 10 ++ outputs/aarch64-linux/tests/btrbk/expr.nix | 15 +++ outputs/x86_64-linux/tests/btrbk/expected.nix | 10 ++ outputs/x86_64-linux/tests/btrbk/expr.nix | 15 +++ 12 files changed, 166 insertions(+), 33 deletions(-) create mode 100644 outputs/aarch64-linux/tests/btrbk/expected.nix create mode 100644 outputs/aarch64-linux/tests/btrbk/expr.nix create mode 100644 outputs/x86_64-linux/tests/btrbk/expected.nix create mode 100644 outputs/x86_64-linux/tests/btrbk/expr.nix diff --git a/hosts/12kingdoms-shoukei/default.nix b/hosts/12kingdoms-shoukei/default.nix index adc1ada1..5501e1bf 100644 --- a/hosts/12kingdoms-shoukei/default.nix +++ b/hosts/12kingdoms-shoukei/default.nix @@ -29,6 +29,14 @@ in inherit (myvars.networking) nameservers; }; + fileSystems."/btr_pool" = { + device = "/dev/disk/by-uuid/c2e8b249-240e-4eef-bf4e-81e7dbbf4887"; + fsType = "btrfs"; + options = [ "subvolid=5" ]; + }; + + modules.btrbk.enable = true; + # This value determines the NixOS release from which the default # settings for stateful data, like file locations and database versions # on your system were taken. It‘s perfectly fine and recommended to leave diff --git a/hosts/idols-ai/default.nix b/hosts/idols-ai/default.nix index ebf0079d..9a176ba4 100644 --- a/hosts/idols-ai/default.nix +++ b/hosts/idols-ai/default.nix @@ -59,6 +59,8 @@ in services.sunshine.enable = true; services.tuned.ppdSettings.main.default = lib.mkForce "performance"; + modules.btrbk.enable = true; + powerManagement.resumeCommands = '' # Insta360 Link may stay enumerated with a stalled UVC endpoint after S3 resume. ${pkgs.coreutils}/bin/sleep 1 diff --git a/hosts/idols-aquamarine/monitoring/alert_rules/node-exporter.yml b/hosts/idols-aquamarine/monitoring/alert_rules/node-exporter.yml index 481842c4..e47887f3 100644 --- a/hosts/idols-aquamarine/monitoring/alert_rules/node-exporter.yml +++ b/hosts/idols-aquamarine/monitoring/alert_rules/node-exporter.yml @@ -293,6 +293,17 @@ groups: summary: Host systemd service crashed (instance {{ $labels.instance }}) description: "systemd service crashed\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" + - alert: HostBtrbkSnapshotFailed + expr: 'node_systemd_unit_state{name="btrbk-btrbk.service",state="failed"} == 1' + for: 5m + labels: + severity: warning + annotations: + summary: Host btrbk snapshot/backup failed (instance {{ $labels.instance }}) + description: + "btrbk-btrbk.service is in a failed state; btrfs snapshots are not being + created.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" + - alert: HostPhysicalComponentTooHot expr: '((node_hwmon_temp_celsius * ignoring(label) group_left(instance, job, node, sensor) diff --git a/hosts/k8s/kubevirt-shoryu/default.nix b/hosts/k8s/kubevirt-shoryu/default.nix index d9e09c53..34959e0c 100644 --- a/hosts/k8s/kubevirt-shoryu/default.nix +++ b/hosts/k8s/kubevirt-shoryu/default.nix @@ -57,6 +57,8 @@ in k3sModule ]; + modules.btrbk.enable = true; + boot.kernelParams = [ # disable transparent hugepage(allocate hugepages dynamically) "transparent_hugepage=never" diff --git a/hosts/k8s/kubevirt-shushou/default.nix b/hosts/k8s/kubevirt-shushou/default.nix index 2bde1c1e..ed35a6c2 100644 --- a/hosts/k8s/kubevirt-shushou/default.nix +++ b/hosts/k8s/kubevirt-shushou/default.nix @@ -51,6 +51,8 @@ in k3sModule ]; + modules.btrbk.enable = true; + boot.kernelParams = [ # disable transparent hugepage(allocate hugepages dynamically) "transparent_hugepage=never" diff --git a/hosts/k8s/kubevirt-youko/default.nix b/hosts/k8s/kubevirt-youko/default.nix index 1e49bf14..61baf3ca 100644 --- a/hosts/k8s/kubevirt-youko/default.nix +++ b/hosts/k8s/kubevirt-youko/default.nix @@ -51,6 +51,8 @@ in k3sModule ]; + modules.btrbk.enable = true; + boot.kernelParams = [ # disable transparent hugepage(allocate hugepages dynamically) "transparent_hugepage=never" diff --git a/modules/nixos/base/btrbk.nix b/modules/nixos/base/btrbk.nix index 2e8a6714..4fd7f839 100644 --- a/modules/nixos/base/btrbk.nix +++ b/modules/nixos/base/btrbk.nix @@ -1,47 +1,103 @@ +{ + config, + lib, + ... +}: +let + cfg = config.modules.btrbk; +in { # ================================================================== # - # Tool for creating snapshots and remote backups of btrfs subvolumes + # btrbk - scheduled LOCAL btrfs snapshots. # https://github.com/digint/btrbk # - # Usage: - # 1. btrbk will create snapshots on schedule - # 2. we can use `btrbk run` command to create a backup manually + # Snapshots are created in `snapshotDir` (relative to the btrfs top-level + # subvolume mounted at `volume`) and pruned by `snapshot_preserve`. With the + # defaults they land in the @snapshots subvolume (mounted at /snapshots): # - # How to restore a snapshot: - # 1. Find the snapshot you want to restore in /snapshots - # 2. Use `btrfs subvol delete /btr_pool/@persistent` to delete the current subvolume - # 3. Use `btrfs subvol snapshot /snapshots/2021-01-01 /btr_pool/@persistent` to restore the snapshot - # 4. reboot the system or remount the filesystem to see the changes + # /btr_pool/@snapshots/@persistent. + # + # These are same-filesystem snapshots: they protect against accidental + # deletion and bad edits, NOT against disk loss. Set `target` (plus + # `services.btrbk.sshAccess` on the receiving host) for off-host backups. + # + # The host MUST mount the btrfs top-level subvolume (subvolid=5) at `volume`; + # this is enforced by an assertion so a missing mount fails evaluation instead + # of failing silently at 3am. + # + # Restore a snapshot (offline; stop writers first): + # 1. btrfs subvolume delete /btr_pool/@persistent + # 2. btrfs subvolume snapshot /btr_pool/@snapshots/@persistent. \ + # /btr_pool/@persistent + # 3. reboot, or remount /persistent, to pick up the restored subvolume. # # ================================================================== + options.modules.btrbk = { + enable = lib.mkEnableOption "scheduled btrfs snapshots via btrbk"; - services.btrbk.instances.btrbk = { - # How often this btrbk instance is started. See systemd.time(7) for more information about the format. - onCalendar = "Tue,Sat *-*-* 3:45:20"; - settings = { - # how to prune local snapshots: - # 1. keep daily snapshots for xx days - snapshot_preserve = "7d"; - # 2. keep all snapshots for 2 days, no matter how frequently you (or your cron job) run btrbk - snapshot_preserve_min = "2d"; + volume = lib.mkOption { + type = lib.types.str; + default = "/btr_pool"; + description = "Mount point of the btrfs top-level subvolume (subvolid=5)."; + }; - # hot to prune remote incremental baqckups: - # keep daily backups for 9 days, weekly backups for 4 weeks, and monthly backups for 2 months - target_preserve = "9d 4w 2m"; - target_preserve_min = "no"; + subvolume = lib.mkOption { + type = lib.types.str; + default = "@persistent"; + description = "Source subvolume to snapshot, relative to {option}`modules.btrbk.volume`."; + }; - volume = { - "/btr_pool" = { - subvolume = { - "@persistent" = { - snapshot_create = "always"; - }; + snapshotDir = lib.mkOption { + type = lib.types.str; + default = "@snapshots"; + description = "Directory the snapshots are created in, relative to {option}`modules.btrbk.volume`."; + }; + + target = lib.mkOption { + type = lib.types.nullOr lib.types.str; + default = null; + description = '' + Optional off-host btrbk backup target (`target` in btrbk.conf). When + null, only local snapshots are kept, which does not protect against + disk loss. + ''; + }; + + onCalendar = lib.mkOption { + type = lib.types.str; + default = "Tue,Sat *-*-* 3:45:20"; + description = "systemd calendar expression for the snapshot timer."; + }; + }; + + config = lib.mkIf cfg.enable { + assertions = [ + { + assertion = builtins.hasAttr cfg.volume config.fileSystems; + message = "modules.btrbk: no filesystem is mounted at `${cfg.volume}`; mount the btrfs top-level subvolume (subvolid=5) there so btrbk can snapshot `${cfg.volume}/${cfg.subvolume}`."; + } + ]; + + services.btrbk.instances.btrbk = { + onCalendar = cfg.onCalendar; + settings = { + # keep daily snapshots for 7 days, and always keep 2 days worth. + snapshot_preserve = "7d"; + snapshot_preserve_min = "2d"; + + # retention for an optional off-host target. + target_preserve = "9d 4w 2m"; + target_preserve_min = "no"; + + volume.${cfg.volume} = { + snapshot_dir = cfg.snapshotDir; + subvolume.${cfg.subvolume} = { + snapshot_create = "always"; }; - - # backup to a remote server or a local directory - # its prune policy is defined by `target_preserve` and `target_preserve_min` - # target = "/snapshots"; + } + // lib.optionalAttrs (cfg.target != null) { + target = cfg.target; }; }; }; diff --git a/modules/nixos/server/qemu-guest.nix b/modules/nixos/server/qemu-guest.nix index aad00957..a9817a45 100644 --- a/modules/nixos/server/qemu-guest.nix +++ b/modules/nixos/server/qemu-guest.nix @@ -20,7 +20,7 @@ config = { # disable backups in the VM - services.btrbk.instances = lib.mkForce { }; + modules.btrbk.enable = lib.mkForce false; boot.growPartition = true; boot.kernelParams = [ "console=ttyS0" ]; diff --git a/outputs/aarch64-linux/tests/btrbk/expected.nix b/outputs/aarch64-linux/tests/btrbk/expected.nix new file mode 100644 index 00000000..4f6c4b3b --- /dev/null +++ b/outputs/aarch64-linux/tests/btrbk/expected.nix @@ -0,0 +1,10 @@ +{ ... }: +{ + enabled = true; + hasBtrPool = true; + onCalendar = "Tue,Sat *-*-* 3:45:20"; + snapshotDir = "@snapshots"; + sources = [ "@persistent" ]; + snapshotCreate = "always"; + hasTarget = false; +} diff --git a/outputs/aarch64-linux/tests/btrbk/expr.nix b/outputs/aarch64-linux/tests/btrbk/expr.nix new file mode 100644 index 00000000..cee61a87 --- /dev/null +++ b/outputs/aarch64-linux/tests/btrbk/expr.nix @@ -0,0 +1,15 @@ +{ outputs, ... }: +let + cfg = outputs.nixosConfigurations.shoukei-niri.config; + instance = cfg.services.btrbk.instances.btrbk; + volume = instance.settings.volume."/btr_pool"; +in +{ + enabled = cfg.modules.btrbk.enable; + hasBtrPool = cfg.fileSystems ? "/btr_pool"; + onCalendar = instance.onCalendar; + snapshotDir = volume.snapshot_dir; + sources = builtins.attrNames volume.subvolume; + snapshotCreate = volume.subvolume."@persistent".snapshot_create; + hasTarget = volume ? target; +} diff --git a/outputs/x86_64-linux/tests/btrbk/expected.nix b/outputs/x86_64-linux/tests/btrbk/expected.nix new file mode 100644 index 00000000..4f6c4b3b --- /dev/null +++ b/outputs/x86_64-linux/tests/btrbk/expected.nix @@ -0,0 +1,10 @@ +{ ... }: +{ + enabled = true; + hasBtrPool = true; + onCalendar = "Tue,Sat *-*-* 3:45:20"; + snapshotDir = "@snapshots"; + sources = [ "@persistent" ]; + snapshotCreate = "always"; + hasTarget = false; +} diff --git a/outputs/x86_64-linux/tests/btrbk/expr.nix b/outputs/x86_64-linux/tests/btrbk/expr.nix new file mode 100644 index 00000000..e2246d03 --- /dev/null +++ b/outputs/x86_64-linux/tests/btrbk/expr.nix @@ -0,0 +1,15 @@ +{ outputs, ... }: +let + cfg = outputs.nixosConfigurations.ai-niri.config; + instance = cfg.services.btrbk.instances.btrbk; + volume = instance.settings.volume."/btr_pool"; +in +{ + enabled = cfg.modules.btrbk.enable; + hasBtrPool = cfg.fileSystems ? "/btr_pool"; + onCalendar = instance.onCalendar; + snapshotDir = volume.snapshot_dir; + sources = builtins.attrNames volume.subvolume; + snapshotCreate = volume.subvolume."@persistent".snapshot_create; + hasTarget = volume ? target; +}