feat(btrbk): make snapshots explicit, guarded, and stored in @snapshots (#271)

* feat(btrbk): make snapshots explicit and guarded

Replace the always-on, hardcoded module with modules.btrbk options (enable/volume/subvolume/snapshotDir/target/onCalendar), and assert the btrfs top-level subvolume is mounted so a missing mount fails evaluation instead of silently failing at runtime.

Store snapshots in the @snapshots subvolume (snapshot_dir=@snapshots) rather than next to the live subvolume, make the off-host target explicit and off by default, and document that local snapshots are not off-host backups.

Enable the feature explicitly on idols-ai, shoukei and the three kubevirt hosts; qemu-guest now disables it through the option. Add eval tests for shoukei and ai.

* feat(monitoring): alert on btrbk snapshot failure

Add HostBtrbkSnapshotFailed on node_systemd_unit_state for btrbk-btrbk.service, so a broken snapshot schedule alerts instead of only surfacing in the journal.

hosts/12kingdoms-shoukei/default.nix

@@ -29,6 +29,14 @@ in
     inherit (myvars.networking) nameservers;
   };
 
+  fileSystems."/btr_pool" = {
+    device = "/dev/disk/by-uuid/c2e8b249-240e-4eef-bf4e-81e7dbbf4887";
+    fsType = "btrfs";
+    options = [ "subvolid=5" ];
+  };
+
+  modules.btrbk.enable = true;
+
   # This value determines the NixOS release from which the default
   # settings for stateful data, like file locations and database versions
   # on your system were taken. It‘s perfectly fine and recommended to leave

hosts/idols-ai/default.nix

@@ -59,6 +59,8 @@ in
   services.sunshine.enable = true;
   services.tuned.ppdSettings.main.default = lib.mkForce "performance";
 
+  modules.btrbk.enable = true;
+
   powerManagement.resumeCommands = ''
     # Insta360 Link may stay enumerated with a stalled UVC endpoint after S3 resume.
     ${pkgs.coreutils}/bin/sleep 1

hosts/idols-aquamarine/monitoring/alert_rules/node-exporter.yml

@@ -293,6 +293,17 @@ groups:
           summary: Host systemd service crashed (instance {{ $labels.instance }})
           description: "systemd service crashed\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
 
+      - alert: HostBtrbkSnapshotFailed
+        expr: 'node_systemd_unit_state{name="btrbk-btrbk.service",state="failed"} == 1'
+        for: 5m
+        labels:
+          severity: warning
+        annotations:
+          summary: Host btrbk snapshot/backup failed (instance {{ $labels.instance }})
+          description:
+            "btrbk-btrbk.service is in a failed state; btrfs snapshots are not being
+            created.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+
       - alert: HostPhysicalComponentTooHot
         expr:
           '((node_hwmon_temp_celsius * ignoring(label) group_left(instance, job, node, sensor)

hosts/k8s/kubevirt-shoryu/default.nix

@@ -57,6 +57,8 @@ in
     k3sModule
   ];
 
+  modules.btrbk.enable = true;
+
   boot.kernelParams = [
     # disable transparent hugepage(allocate hugepages dynamically)
     "transparent_hugepage=never"

hosts/k8s/kubevirt-shushou/default.nix

@@ -51,6 +51,8 @@ in
     k3sModule
   ];
 
+  modules.btrbk.enable = true;
+
   boot.kernelParams = [
     # disable transparent hugepage(allocate hugepages dynamically)
     "transparent_hugepage=never"

hosts/k8s/kubevirt-youko/default.nix

@@ -51,6 +51,8 @@ in
     k3sModule
   ];
 
+  modules.btrbk.enable = true;
+
   boot.kernelParams = [
     # disable transparent hugepage(allocate hugepages dynamically)
     "transparent_hugepage=never"

modules/nixos/base/btrbk.nix

@@ -1,47 +1,103 @@
+{
+  config,
+  lib,
+  ...
+}:
+let
+  cfg = config.modules.btrbk;
+in
 {
   # ==================================================================
   #
-  # Tool for creating snapshots and remote backups of btrfs subvolumes
+  # btrbk - scheduled LOCAL btrfs snapshots.
   #   https://github.com/digint/btrbk
   #
-  # Usage:
+  # Snapshots are created in `snapshotDir` (relative to the btrfs top-level
-  #   1. btrbk will create snapshots on schedule
+  # subvolume mounted at `volume`) and pruned by `snapshot_preserve`. With the
-  #   2. we can use `btrbk run` command to create a backup manually
+  # defaults they land in the @snapshots subvolume (mounted at /snapshots):
   #
-  # How to restore a snapshot:
+  #   /btr_pool/@snapshots/@persistent.<timestamp>
-  #   1. Find the snapshot you want to restore in /snapshots
+  #
-  #   2. Use `btrfs subvol delete /btr_pool/@persistent` to delete the current subvolume
+  # These are same-filesystem snapshots: they protect against accidental
-  #   3. Use `btrfs subvol snapshot /snapshots/2021-01-01 /btr_pool/@persistent` to restore the snapshot
+  # deletion and bad edits, NOT against disk loss. Set `target` (plus
-  #   4. reboot the system or remount the filesystem to see the changes
+  # `services.btrbk.sshAccess` on the receiving host) for off-host backups.
+  #
+  # The host MUST mount the btrfs top-level subvolume (subvolid=5) at `volume`;
+  # this is enforced by an assertion so a missing mount fails evaluation instead
+  # of failing silently at 3am.
+  #
+  # Restore a snapshot (offline; stop writers first):
+  #   1. btrfs subvolume delete /btr_pool/@persistent
+  #   2. btrfs subvolume snapshot /btr_pool/@snapshots/@persistent.<timestamp> \
+  #        /btr_pool/@persistent
+  #   3. reboot, or remount /persistent, to pick up the restored subvolume.
   #
   # ==================================================================
+  options.modules.btrbk = {
+    enable = lib.mkEnableOption "scheduled btrfs snapshots via btrbk";
 
-  services.btrbk.instances.btrbk = {
+    volume = lib.mkOption {
-    # How often this btrbk instance is started. See systemd.time(7) for more information about the format.
+      type = lib.types.str;
-    onCalendar = "Tue,Sat *-*-* 3:45:20";
+      default = "/btr_pool";
-    settings = {
+      description = "Mount point of the btrfs top-level subvolume (subvolid=5).";
-      # how to prune local snapshots:
+    };
-      # 1. keep daily snapshots for xx days
-      snapshot_preserve = "7d";
-      # 2. keep all snapshots for 2 days, no matter how frequently you (or your cron job) run btrbk
-      snapshot_preserve_min = "2d";
 
-      # hot to prune remote incremental baqckups:
+    subvolume = lib.mkOption {
-      # keep daily backups for 9 days, weekly backups for 4 weeks, and monthly backups for 2 months
+      type = lib.types.str;
-      target_preserve = "9d 4w 2m";
+      default = "@persistent";
-      target_preserve_min = "no";
+      description = "Source subvolume to snapshot, relative to {option}`modules.btrbk.volume`.";
+    };
 
-      volume = {
+    snapshotDir = lib.mkOption {
-        "/btr_pool" = {
+      type = lib.types.str;
-          subvolume = {
+      default = "@snapshots";
-            "@persistent" = {
+      description = "Directory the snapshots are created in, relative to {option}`modules.btrbk.volume`.";
-              snapshot_create = "always";
+    };
-            };
+
+    target = lib.mkOption {
+      type = lib.types.nullOr lib.types.str;
+      default = null;
+      description = ''
+        Optional off-host btrbk backup target (`target` in btrbk.conf). When
+        null, only local snapshots are kept, which does not protect against
+        disk loss.
+      '';
+    };
+
+    onCalendar = lib.mkOption {
+      type = lib.types.str;
+      default = "Tue,Sat *-*-* 3:45:20";
+      description = "systemd calendar expression for the snapshot timer.";
+    };
+  };
+
+  config = lib.mkIf cfg.enable {
+    assertions = [
+      {
+        assertion = builtins.hasAttr cfg.volume config.fileSystems;
+        message = "modules.btrbk: no filesystem is mounted at `${cfg.volume}`; mount the btrfs top-level subvolume (subvolid=5) there so btrbk can snapshot `${cfg.volume}/${cfg.subvolume}`.";
+      }
+    ];
+
+    services.btrbk.instances.btrbk = {
+      onCalendar = cfg.onCalendar;
+      settings = {
+        # keep daily snapshots for 7 days, and always keep 2 days worth.
+        snapshot_preserve = "7d";
+        snapshot_preserve_min = "2d";
+
+        # retention for an optional off-host target.
+        target_preserve = "9d 4w 2m";
+        target_preserve_min = "no";
+
+        volume.${cfg.volume} = {
+          snapshot_dir = cfg.snapshotDir;
+          subvolume.${cfg.subvolume} = {
+            snapshot_create = "always";
           };
-
+        }
-          # backup to a remote server or a local directory
+        // lib.optionalAttrs (cfg.target != null) {
-          # its prune policy is defined by `target_preserve` and `target_preserve_min`
+          target = cfg.target;
-          # target = "/snapshots";
         };
       };
     };

modules/nixos/server/qemu-guest.nix

@@ -20,7 +20,7 @@
 
   config = {
     # disable backups in the VM
-    services.btrbk.instances = lib.mkForce { };
+    modules.btrbk.enable = lib.mkForce false;
 
     boot.growPartition = true;
     boot.kernelParams = [ "console=ttyS0" ];

outputs/aarch64-linux/tests/btrbk/expected.nix

@@ -0,0 +1,10 @@
+{ ... }:
+{
+  enabled = true;
+  hasBtrPool = true;
+  onCalendar = "Tue,Sat *-*-* 3:45:20";
+  snapshotDir = "@snapshots";
+  sources = [ "@persistent" ];
+  snapshotCreate = "always";
+  hasTarget = false;
+}

outputs/aarch64-linux/tests/btrbk/expr.nix

@@ -0,0 +1,15 @@
+{ outputs, ... }:
+let
+  cfg = outputs.nixosConfigurations.shoukei-niri.config;
+  instance = cfg.services.btrbk.instances.btrbk;
+  volume = instance.settings.volume."/btr_pool";
+in
+{
+  enabled = cfg.modules.btrbk.enable;
+  hasBtrPool = cfg.fileSystems ? "/btr_pool";
+  onCalendar = instance.onCalendar;
+  snapshotDir = volume.snapshot_dir;
+  sources = builtins.attrNames volume.subvolume;
+  snapshotCreate = volume.subvolume."@persistent".snapshot_create;
+  hasTarget = volume ? target;
+}

outputs/x86_64-linux/tests/btrbk/expected.nix

@@ -0,0 +1,10 @@
+{ ... }:
+{
+  enabled = true;
+  hasBtrPool = true;
+  onCalendar = "Tue,Sat *-*-* 3:45:20";
+  snapshotDir = "@snapshots";
+  sources = [ "@persistent" ];
+  snapshotCreate = "always";
+  hasTarget = false;
+}

outputs/x86_64-linux/tests/btrbk/expr.nix

@@ -0,0 +1,15 @@
+{ outputs, ... }:
+let
+  cfg = outputs.nixosConfigurations.ai-niri.config;
+  instance = cfg.services.btrbk.instances.btrbk;
+  volume = instance.settings.volume."/btr_pool";
+in
+{
+  enabled = cfg.modules.btrbk.enable;
+  hasBtrPool = cfg.fileSystems ? "/btr_pool";
+  onCalendar = instance.onCalendar;
+  snapshotDir = volume.snapshot_dir;
+  sources = builtins.attrNames volume.subvolume;
+  snapshotCreate = volume.subvolume."@persistent".snapshot_create;
+  hasTarget = volume ? target;
+}