Merge pull request #75 from ryan4yin/aarch-tmpfs-on-root

feat: aarch64 - tmpfs on root + impermanence

hosts/12kingdoms_rakushun/README.md

@@ -21,11 +21,41 @@ nvme0n1     259:0    0  1.8T  0 disk
   └─crypted 253:0    0  1.8T  0 crypt /tmp
                                       /swap
                                       /snapshots
-                                      /home
+                                      /home/ryan/tmp
+                                      /home/ryan/nix-config
+                                      /home/ryan/go
+                                      /home/ryan/codes
+                                      /home/ryan/.ssh
+                                      /home/ryan/.local/state
+                                      /home/ryan/.npm
+                                      /home/ryan/.local/share
+                                      /home/ryan/.conda
+                                      /etc/ssh
+                                      /etc/nix/inputs
+                                      /etc/secureboot
+                                      /etc/agenix
+                                      /etc/NetworkManager/system-connections
+                                      /etc/machine-id
                                       /nix/store
+                                      /var/log
                                       /var/lib
                                       /nix
-                                      /
+                                      /persistent
+
+[ryan@rakushun:~]$ df -Th
+Filesystem          Type      Size  Used Avail Use% Mounted on
+devtmpfs            devtmpfs  785M     0  785M   0% /dev
+tmpfs               tmpfs     7.7G     0  7.7G   0% /dev/shm
+tmpfs               tmpfs     3.9G  6.8M  3.9G   1% /run
+tmpfs               tmpfs     7.7G  1.9M  7.7G   1% /run/wrappers
+none                tmpfs     4.0G   48K  4.0G   1% /
+/dev/mapper/crypted btrfs     1.9T   19G  1.8T   2% /persistent
+/dev/mapper/crypted btrfs     1.9T   19G  1.8T   2% /nix
+/dev/mapper/crypted btrfs     1.9T   19G  1.8T   2% /snapshots
+/dev/mapper/crypted btrfs     1.9T   19G  1.8T   2% /swap
+/dev/mapper/crypted btrfs     1.9T   19G  1.8T   2% /tmp
+/dev/nvme0n1p1      vfat      629M   96M  534M  16% /boot
+tmpfs               tmpfs     1.6G  4.0K  1.6G   1% /run/user/1000
 ```
 
 CPU info:

hosts/12kingdoms_rakushun/default.nix

@@ -17,8 +17,9 @@ in {
     # import the rk3588 module, which contains the configuration for bootloader/kernel/firmware
     nixos-rk3588.nixosModules.orangepi5plus.core
     disko.nixosModules.default
-    ./disko-fs.nix
     ./hardware-configuration.nix
+    ./disko-fs.nix
+    ./impermanence.nix
   ];
 
   networking = {

hosts/12kingdoms_rakushun/disko-fs.nix

@@ -1,6 +1,20 @@
 {
+  # required by impermanence
+  fileSystems."/persistent".neededForBoot = true;
+
   disko.devices = {
-    # TODO: rename to nvme0n1
+    nodev."/" = {
+      fsType = "tmpfs";
+      mountOptions = [
+        "size=4G"
+        "defaults"
+        # set mode to 755, otherwise systemd will set it to 777, which cause problems.
+        # relatime: Update inode access times relative to modify or change time.
+        "mode=755"
+      ];
+    };
+
+    # TODO: rename to main
     disk.sda = {
       type = "disk";
       # When using disko-install, we will overwrite this value from the commandline
@@ -57,24 +71,14 @@
                 type = "btrfs";
                 extraArgs = ["-f"];
                 subvolumes = {
-                  # TODO: tmpfs on root
-                  "@root" = {
-                    mountpoint = "/";
-                    mountOptions = ["compress-force=zstd:1" "noatime"];
-                  };
-                  "@home" = {
-                    mountpoint = "/home";
-                    mountOptions = ["compress-force=zstd:1"];
-                  };
-                  "@lib" = {
-                    mountpoint = "/var/lib";
-                    mountOptions = ["compress-force=zstd:1"];
-                  };
-
                   "@nix" = {
                     mountpoint = "/nix";
                     mountOptions = ["compress-force=zstd:1" "noatime"];
                   };
+                  "@persistent" = {
+                    mountpoint = "/persistent";
+                    mountOptions = ["compress-force=zstd:1" "noatime"];
+                  };
                   "@tmp" = {
                     mountpoint = "/tmp";
                     mountOptions = ["compress-force=zstd:1" "noatime"];

hosts/12kingdoms_rakushun/impermanence.nix

@@ -0,0 +1,75 @@
+{
+  impermanence,
+  pkgs,
+  ...
+}: {
+  imports = [
+    impermanence.nixosModules.impermanence
+  ];
+
+  environment.systemPackages = [
+    # `sudo ncdu -x /`
+    pkgs.ncdu
+  ];
+
+  # There are two ways to clear the root filesystem on every boot:
+  ##  1. use tmpfs for /
+  ##  2. (btrfs/zfs only)take a blank snapshot of the root filesystem and revert to it on every boot via:
+  ##     boot.initrd.postDeviceCommands = ''
+  ##       mkdir -p /run/mymount
+  ##       mount -o subvol=/ /dev/disk/by-uuid/UUID /run/mymount
+  ##       btrfs subvolume delete /run/mymount
+  ##       btrfs subvolume snapshot / /run/mymount
+  ##     '';
+  #
+  #  See also https://grahamc.com/blog/erase-your-darlings/
+
+  # NOTE: impermanence only mounts the directory/file list below to /persistent
+  # If the directory/file already exists in the root filesystem, you should
+  # move those files/directories to /persistent first!
+  environment.persistence."/persistent" = {
+    # sets the mount option x-gvfs-hide on all the bind mounts
+    # to hide them from the file manager
+    hideMounts = true;
+    directories = [
+      "/etc/NetworkManager/system-connections"
+      "/etc/ssh"
+      "/etc/nix/inputs"
+      "/etc/secureboot" # lanzaboote - secure boot
+      # my secrets
+      "/etc/agenix/"
+
+      "/var/log"
+      "/var/lib"
+    ];
+    files = [
+      "/etc/machine-id"
+    ];
+
+    # the following directories will be passed to /persistent/home/$USER
+    users.ryan = {
+      directories = [
+        "codes"
+        "nix-config"
+        "tmp"
+
+        {
+          directory = ".ssh";
+          mode = "0700";
+        }
+
+        # neovim / remmina / flatpak / ...
+        ".local/share"
+        ".local/state"
+
+        # language package managers
+        ".npm"
+        ".conda" # generated by `conda-shell`
+        "go"
+      ];
+      files = [
+        ".config/nushell/history.txt"
+      ];
+    };
+  };
+}

hosts/12kingdoms_suzu/README.md

@@ -22,11 +22,40 @@ nvme0n1     259:0    0 238.5G  0 disk
   └─crypted 253:0    0 237.8G  0 crypt /tmp
                                        /snapshots
                                        /swap
-                                       /home
+                                       /home/ryan/tmp
+                                       /home/ryan/nix-config
+                                       /home/ryan/go
+                                       /home/ryan/.local/state
+                                       /home/ryan/codes
+                                       /home/ryan/.npm
+                                       /home/ryan/.ssh
+                                       /home/ryan/.local/share
+                                       /etc/ssh
+                                       /home/ryan/.conda
+                                       /etc/secureboot
+                                       /etc/agenix
+                                       /etc/nix/inputs
+                                       /etc/NetworkManager/system-connections
                                        /nix/store
+                                       /var/log
                                        /var/lib
                                        /nix
-                                       /
+                                       /persistent
+
+[ryan@suzu:~]$ df -Th
+Filesystem          Type      Size  Used Avail Use% Mounted on
+devtmpfs            devtmpfs  383M     0  383M   0% /dev
+tmpfs               tmpfs     3.8G     0  3.8G   0% /dev/shm
+tmpfs               tmpfs     1.9G  6.2M  1.9G   1% /run
+tmpfs               tmpfs     3.8G  1.9M  3.8G   1% /run/wrappers
+none                tmpfs     2.0G   48K  2.0G   1% /
+/dev/mapper/crypted btrfs     238G   11G  226G   5% /persistent
+/dev/mapper/crypted btrfs     238G   11G  226G   5% /nix
+/dev/mapper/crypted btrfs     238G   11G  226G   5% /swap
+/dev/mapper/crypted btrfs     238G   11G  226G   5% /snapshots
+/dev/mapper/crypted btrfs     238G   11G  226G   5% /tmp
+/dev/nvme0n1p1      vfat      629M   86M  543M  14% /boot
+tmpfs               tmpfs     766M  4.0K  766M   1% /run/user/1000
 ```
 
 CPU info:

hosts/12kingdoms_suzu/default.nix

@@ -17,8 +17,9 @@ in {
     # import the rk3588 module, which contains the configuration for bootloader/kernel/firmware
     nixos-rk3588.nixosModules.orangepi5plus.core
     disko.nixosModules.default
-    ./disko-fs.nix
     ./hardware-configuration.nix
+    ./disko-fs.nix
+    ./impermanence.nix
   ];
 
   networking = {

hosts/12kingdoms_suzu/disko-fs.nix

@@ -1,6 +1,20 @@
 {
+  # required by impermanence
+  fileSystems."/persistent".neededForBoot = true;
+
   disko.devices = {
-    # TODO: rename to nvme0n1
+    nodev."/" = {
+      fsType = "tmpfs";
+      mountOptions = [
+        "size=2G"
+        "defaults"
+        # set mode to 755, otherwise systemd will set it to 777, which cause problems.
+        # relatime: Update inode access times relative to modify or change time.
+        "mode=755"
+      ];
+    };
+
+    # TODO: rename to main
     disk.sda = {
       type = "disk";
       # When using disko-install, we will overwrite this value from the commandline
@@ -57,24 +71,14 @@
                 type = "btrfs";
                 extraArgs = ["-f"];
                 subvolumes = {
-                  # TODO: tmpfs on root
-                  "@root" = {
-                    mountpoint = "/";
-                    mountOptions = ["compress-force=zstd:1" "noatime"];
-                  };
-                  "@home" = {
-                    mountpoint = "/home";
-                    mountOptions = ["compress-force=zstd:1"];
-                  };
-                  "@lib" = {
-                    mountpoint = "/var/lib";
-                    mountOptions = ["compress-force=zstd:1"];
-                  };
-
                   "@nix" = {
                     mountpoint = "/nix";
                     mountOptions = ["compress-force=zstd:1" "noatime"];
                   };
+                  "@persistent" = {
+                    mountpoint = "/persistent";
+                    mountOptions = ["compress-force=zstd:1" "noatime"];
+                  };
                   "@tmp" = {
                     mountpoint = "/tmp";
                     mountOptions = ["compress-force=zstd:1" "noatime"];

hosts/12kingdoms_suzu/impermanence.nix

@@ -0,0 +1,75 @@
+{
+  impermanence,
+  pkgs,
+  ...
+}: {
+  imports = [
+    impermanence.nixosModules.impermanence
+  ];
+
+  environment.systemPackages = [
+    # `sudo ncdu -x /`
+    pkgs.ncdu
+  ];
+
+  # There are two ways to clear the root filesystem on every boot:
+  ##  1. use tmpfs for /
+  ##  2. (btrfs/zfs only)take a blank snapshot of the root filesystem and revert to it on every boot via:
+  ##     boot.initrd.postDeviceCommands = ''
+  ##       mkdir -p /run/mymount
+  ##       mount -o subvol=/ /dev/disk/by-uuid/UUID /run/mymount
+  ##       btrfs subvolume delete /run/mymount
+  ##       btrfs subvolume snapshot / /run/mymount
+  ##     '';
+  #
+  #  See also https://grahamc.com/blog/erase-your-darlings/
+
+  # NOTE: impermanence only mounts the directory/file list below to /persistent
+  # If the directory/file already exists in the root filesystem, you should
+  # move those files/directories to /persistent first!
+  environment.persistence."/persistent" = {
+    # sets the mount option x-gvfs-hide on all the bind mounts
+    # to hide them from the file manager
+    hideMounts = true;
+    directories = [
+      "/etc/NetworkManager/system-connections"
+      "/etc/ssh"
+      "/etc/nix/inputs"
+      "/etc/secureboot" # lanzaboote - secure boot
+      # my secrets
+      "/etc/agenix/"
+
+      "/var/log"
+      "/var/lib"
+    ];
+    files = [
+      "/etc/machine-id"
+    ];
+
+    # the following directories will be passed to /persistent/home/$USER
+    users.ryan = {
+      directories = [
+        "codes"
+        "nix-config"
+        "tmp"
+
+        {
+          directory = ".ssh";
+          mode = "0700";
+        }
+
+        # neovim / remmina / flatpak / ...
+        ".local/share"
+        ".local/state"
+
+        # language package managers
+        ".npm"
+        ".conda" # generated by `conda-shell`
+        "go"
+      ];
+      files = [
+        ".config/nushell/history.txt"
+      ];
+    };
+  };
+}