fix(preservation): mount /etc/ssh in initrd so key changes apply on boot

NixOS copies mode-carrying /etc entries such as authorized_keys.d during
stage-2 activation, which runs before systemd starts. preservation mounted the
persisted /etc/ssh from preservation.target afterwards and hid that copy, so a
key change only took effect on `switch`: revoking a key and rebooting silently
restored the old one.

Prepare the mount in initrd, before activation runs, and cover it with a VM
test that seeds a stale authorized_keys file, reboots, and asserts the declared
key replaced it. The same test fails on that assertion with inInitrd disabled.

Verified on all four hosts: /persistent/etc/ssh/authorized_keys.d/root is now
rewritten at boot instead of keeping the timestamp of the last switch.

Assisted-by: Claude Code (claude-opus-5-5)

hosts/12kingdoms-shoryu/preservation.nix

@@ -25,7 +25,15 @@ in
   preservation.preserveAt."/persistent" = {
     directories = [
       "/etc/NetworkManager/system-connections"
-      "/etc/ssh"
+      # Mount in initrd: NixOS copies mode-carrying /etc entries (such as
+      # authorized_keys.d) during stage-2 activation, which runs before systemd
+      # starts. A mount created by preservation.target comes later and would
+      # hide those copies behind the previous `switch`, so key changes only
+      # applied on `switch`, never on `boot` + reboot.
+      {
+        directory = "/etc/ssh";
+        inInitrd = true;
+      }
       "/etc/nix/inputs"
       "/etc/secureboot" # lanzaboote - secure boot
       # my secrets

hosts/idols-aquamarine/preservation.nix

@@ -24,8 +24,17 @@ in
   # If the directory/file already exists in the root filesystem you should
   # move those files/directories to /persistent first!
   preservation.preserveAt."/persistent" = {
+    # Mount in initrd: NixOS copies mode-carrying /etc entries (such as
+    # authorized_keys.d) during stage-2 activation, which runs before systemd
+    # starts. A mount created by preservation.target comes later and would hide
+    # those copies behind the previous `switch`, so key changes only applied on
+    # `switch`, never on `boot` + reboot. Preparing the mount in initrd puts
+    # /persistent/etc/ssh at /sysroot/etc/ssh before activation runs.
     directories = [
-      "/etc/ssh"
+      {
+        directory = "/etc/ssh";
+        inInitrd = true;
+      }
       "/etc/nix/inputs"
       "/etc/secureboot" # lanzaboote - secure boot (see ./secureboot.nix)
 

outputs/default.nix

@@ -220,6 +220,12 @@ in
         pkgs = nixpkgs.legacyPackages.${system};
         inherit (inputs) agenix;
       };
+      # A persisted /etc/ssh must receive the activation copy at boot, which
+      # only happens when preservation prepares the mount in initrd.
+      security-preservation-etc = import ../tests/security-preservation-etc.nix {
+        pkgs = nixpkgs.legacyPackages.${system};
+        inherit (inputs) preservation;
+      };
     }
   );
 

tests/security-preservation-etc.nix

@@ -0,0 +1,70 @@
+{ pkgs, preservation }:
+# `/etc` entries that carry a mode are *copied* into /etc by stage-2
+# activation (setup-etc.pl), which runs before systemd. preservation mounts a
+# persisted directory only after systemd starts, so a persisted /etc/ssh
+# shadows the fresh copies and sshd keeps reading the previous `switch`.
+#
+# This boots a VM with /etc/ssh persisted from a separate disk and checks that
+# the declared authorized_keys file reached the persistent volume, which only
+# happens when the mount is prepared in initrd.
+pkgs.testers.runNixOSTest {
+  name = "security-preservation-etc";
+
+  nodes.machine =
+    { lib, ... }:
+    {
+      imports = [ preservation.nixosModules.default ];
+
+      virtualisation.emptyDiskImages = [ 256 ];
+      virtualisation.memorySize = 1024;
+
+      fileSystems."/persistent" = {
+        device = "/dev/vdb";
+        fsType = "ext4";
+        autoFormat = true;
+        neededForBoot = true;
+      };
+
+      preservation.enable = true;
+      preservation.preserveAt."/persistent".directories = [
+        {
+          directory = "/etc/ssh";
+          inInitrd = true;
+        }
+      ];
+
+      services.openssh.enable = true;
+      users.users.alice = {
+        isNormalUser = true;
+        openssh.authorizedKeys.keys = [
+          "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAITestOnlyKeyMaterial000000000000000000000 test@example"
+        ];
+      };
+    };
+
+  testScript = ''
+    # First boot only to get the persistent volume formatted.
+    machine.start()
+    machine.wait_for_unit("multi-user.target")
+
+    # Seed a stale authorized_keys file, as a previous `switch` would have left
+    # it, then reboot into the same generation with a different declared key.
+    machine.succeed("mkdir -p /persistent/etc/ssh/authorized_keys.d")
+    machine.succeed("echo 'ssh-ed25519 AAAAOld stale@example' > /persistent/etc/ssh/authorized_keys.d/alice")
+    machine.succeed("chmod 444 /persistent/etc/ssh/authorized_keys.d/alice")
+    machine.shutdown()
+
+    machine.start()
+    machine.wait_for_unit("multi-user.target")
+    machine.succeed("mountpoint -q /etc/ssh")
+
+    # The boot-time copy must land on the persistent volume and replace the
+    # stale file; with the mount ordered after activation it stays stale.
+    machine.succeed("grep -q test@example /persistent/etc/ssh/authorized_keys.d/alice")
+    machine.fail("grep -q stale@example /etc/ssh/authorized_keys.d/alice")
+    machine.succeed("grep -q test@example /etc/ssh/authorized_keys.d/alice")
+    machine.succeed("test \"$(stat -c %a /etc/ssh/authorized_keys.d/alice)\" = 444")
+
+    machine.wait_for_unit("sshd.service")
+  '';
+}