diff --git a/hosts/idols-ai/README.md b/hosts/idols-ai/README.md index 8b5be999..390676a4 100644 --- a/hosts/idols-ai/README.md +++ b/hosts/idols-ai/README.md @@ -6,7 +6,10 @@ Desktop (NixOS + preservation, LUKS + btrfs on nvme). Disk layout is declarative Related: - [nixos-installer README](/nixos-installer/README.md) – install from ISO using disko -- [disko-fs.nix](./disko-fs.nix) – partition/layout definition (ESP + LUKS + btrfs) +- [disko-fs.nix](./disko-fs.nix) – main disk layout (ESP + LUKS + btrfs). From + `nix-config/nixos-installer`: + `nix run github:nix-community/disko -- --mode destroy,format,mount ../hosts/idols-ai/disko-fs.nix` +- [disko-fs-media.nix](./disko-fs-media.nix) – media disk layout (LUKS + btrfs at /persistent/media) ## TODOs diff --git a/hosts/idols-ai/default.nix b/hosts/idols-ai/default.nix index e65bfdfc..4d14fed3 100644 --- a/hosts/idols-ai/default.nix +++ b/hosts/idols-ai/default.nix @@ -20,8 +20,11 @@ in { imports = [ disko.nixosModules.default + # disks ./disko-fs.nix + ./disko-fs-data.nix ./netdev-mount.nix + # Include the results of the hardware scan. ./hardware-configuration.nix ./nvidia.nix diff --git a/hosts/idols-ai/disko-fs-data.nix b/hosts/idols-ai/disko-fs-data.nix new file mode 100644 index 00000000..48c51422 --- /dev/null +++ b/hosts/idols-ai/disko-fs-data.nix @@ -0,0 +1,62 @@ +# Disko layout for idols-ai data disk (LUKS + btrfs, mount at /persistent/data). +# +# Destroy, format & mount (wipes disk; from nixos-installer: cd nix-config/nixos-installer): +# nix run github:nix-community/disko -- --mode destroy,format,mount ../hosts/idols-ai/disko-fs-data.nix +# Mount only (after first format): +# nix run github:nix-community/disko -- --mode mount ../hosts/idols-ai/disko-fs-data.nix +# +{ + disko.devices = { + disk.data = { + type = "disk"; + device = "/dev/disk/by-id/nvme-Fanxiang_S790_2TB_FXS790254050582"; + content = { + type = "gpt"; + partitions = { + datapart = { + size = "100%"; + content = { + type = "luks"; + name = "data-luks"; # Mapper name; match boot.initrd.luks + settings = { + allowDiscards = true; # TRIM for SSDs; slightly less secure, better performance + }; + # Add boot.initrd.luks.devices so initrd prompts for passphrase at boot + initrdUnlock = true; + # cryptsetup luksFormat options + extraFormatArgs = [ + "--type luks2" + "--cipher aes-xts-plain64" + "--hash sha512" + "--iter-time 5000" + "--key-size 256" + "--pbkdf argon2id" + "--use-random" # Block until enough entropy from /dev/random + ]; + extraOpenArgs = [ + "--timeout 10" + ]; + content = { + type = "btrfs"; + extraArgs = [ "-f" ]; # Force overwrite if filesystem already exists + subvolumes = { + "@data" = { + mountpoint = "/data"; + mountOptions = [ + "compress-force=zstd:1" + ]; + }; + }; + postMountHook = '' + chown ryan:users /mnt/data + # Set SGID + rwx for owner/group, read-only for others; new files inherit group + chmod 2755 /mnt/data + ''; + }; + }; + }; + }; + }; + }; + }; +} diff --git a/hosts/idols-ai/disko-fs.nix b/hosts/idols-ai/disko-fs.nix index eac1781f..8fc28806 100644 --- a/hosts/idols-ai/disko-fs.nix +++ b/hosts/idols-ai/disko-fs.nix @@ -1,12 +1,12 @@ # Disko layout for idols-ai on nvme1n1 (target disk after migration). # Same structure as current nvme0n1: ESP + LUKS + btrfs with ephemeral root (tmpfs). # -# Format & mount (from installer or live system): -# nix run github:nix-community/disko -- --mode disko ./disko-fs.nix +# Destroy, format & mount (wipes disk; from nixos-installer: cd nix-config/nixos-installer): +# nix run github:nix-community/disko -- --mode destroy,format,mount ../hosts/idols-ai/disko-fs.nix # Mount only (after first format): -# nix run github:nix-community/disko -- --mode mount ./disko-fs.nix +# nix run github:nix-community/disko -- --mode mount ../hosts/idols-ai/disko-fs.nix # -# Use by-id for stability; override device when installing, e.g.: +# Override device when installing, e.g.: # nixos-install --flake .#ai --option disko.devices.disk.nixos-ai.device /dev/nvme1n1 { # Ephemeral root; preservation mounts /persistent for state. @@ -24,8 +24,7 @@ disk.nixos-ai = { type = "disk"; - # Override at install time if needed: --option disko.devices.disk.nixos-ai.device /dev/nvme1n1 - device = "/dev/nvme1n1"; + device = "/dev/disk/by-id/nvme-KINGBANK_KP260_CN092BH3203917"; content = { type = "gpt"; partitions = { diff --git a/hosts/idols-aquamarine/disko-fs.nix b/hosts/idols-aquamarine/disko-fs.nix index 7d11d14f..837b9dce 100644 --- a/hosts/idols-aquamarine/disko-fs.nix +++ b/hosts/idols-aquamarine/disko-fs.nix @@ -1,5 +1,7 @@ -# auto disk partitioning: -# nix run github:nix-community/disko -- --mode disko ./disko-fs.nix +# Auto disk partitioning (from repo root: nix-config): +# nix run github:nix-community/disko -- --mode destroy,format,mount hosts/idols-aquamarine/disko-fs.nix +# Mount only (after first format): +# nix run github:nix-community/disko -- --mode mount hosts/idols-aquamarine/disko-fs.nix let cryptKeyFile = "/etc/agenix/hdd-luks-crypt-key"; unlockDisk = "data-encrypted"; diff --git a/hosts/k8s/disko-config/README.md b/hosts/k8s/disko-config/README.md index 94f26db6..58d5fbae 100644 --- a/hosts/k8s/disko-config/README.md +++ b/hosts/k8s/disko-config/README.md @@ -1,37 +1,6 @@ # Disko Config -Generate LUKS keyfile to encrypt the root partition, it's used by disko. - -```bash -# partition the usb stick -DEV=/dev/sdX -parted $DEV -- mklabel gpt -parted $DEV -- mkpart primary 2M 512MB -parted $DEV -- mkpart primary 512MB 1024MB -mkfs.fat -F 32 -n NIXOS_DSC ${DEV}1 -mkfs.fat -F 32 -n NIXOS_K3S ${DEV}2 - -# Generate a keyfile from the true random number generator -KEYFILE=./kubevirt-luks-keyfile -dd bs=512 count=64 iflag=fullblock if=/dev/random of=$KEYFILE - -# generate token for k3s -K3S_TOKEN_FILE=./kubevirt-k3s-token -K3S_TOKEN=$(grep -ao '[A-Za-z0-9]' < /dev/random | head -64 | tr -d '\n' ; echo "") -echo $K3S_TOKEN > $K3S_TOKEN_FILE - -# copy the keyfile and token to the usb stick -KEYFILE=./kubevirt-luks-keyfile -DEVICE=/dev/disk/by-label/NIXOS_DSC -# seek=128 skip N obs-sized output blocks to avoid overwriting the filesystem header -dd bs=512 count=64 iflag=fullblock seek=128 if=$KEYFILE of=$DEVICE - -K3S_TOKEN_FILE=./kubevirt-k3s-token -USB_PATH=/run/media/ryan/NIXOS_K3S -cp $K3S_TOKEN_FILE $USB_PATH -``` - -### 2. Partition the SSD & install NixOS via disko +## Partition the SSD & install NixOS via disko ```bash # enter an shell with git/vim/ssh-agent/gnumake available @@ -41,15 +10,13 @@ git clone https://github.com/ryan4yin/nix-config.git cd nix-config -# one line -sudo nix run --experimental-features "nix-command flakes" 'github:nix-community/disko#disko-install' -- \ - --write-efi-boot-entries --disk main /dev/nvme0n1 --flake .#kubevirt-shoryu - - -# or step by step ## 1. partition & format the disk via disko -sudo nix --experimental-features "nix-command flakes" run github:nix-community/disko -- --mode disko hosts/k8s/disko-config/kukubevirt-disko-fs.nix -## 2. install nixos +# encrypt the root partition with luks2 and argon2id, will prompt for a passphrase, which will be used to unlock the partition. +sudo nix --experimental-features "nix-command flakes" run github:nix-community/disko -- --mode destroy,format,mount hosts/k8s/disko-config/kubevirt-disko-fs.nix +## 2. setup the automatic unlock via the tpm2 chip +systemd-cryptenroll --tpm2-device=auto --tpm2-pcrs=0+7 /dev/ + +## 3. install nixos sudo nixos-install --root /mnt --no-root-password --show-trace --verbose --flake .#kubevirt-shoryu # enter into the installed system, check password & users @@ -66,4 +33,6 @@ mv /etc/machine-id /persistent/etc/ mv /etc/ssh /persistent/etc/ mkdir -p /persistent/home/ryan chown -R ryan:ryan /persistent/home/ryan + +# add your k3s token at /persistent/kubevirt-k3s-token ``` diff --git a/hosts/k8s/disko-config/kubevirt-disko-fs.nix b/hosts/k8s/disko-config/kubevirt-disko-fs.nix index d75157fc..ab16930c 100644 --- a/hosts/k8s/disko-config/kubevirt-disko-fs.nix +++ b/hosts/k8s/disko-config/kubevirt-disko-fs.nix @@ -2,13 +2,6 @@ # required by preservation fileSystems."/persistent".neededForBoot = true; - # contains the k3s's token - fileSystems."/run/media/nixos_k3s" = { - device = "/dev/disk/by-label/NIXOS_K3S"; - fsType = "vfat"; - options = [ "ro" ]; - }; - disko.devices = { nodev."/" = { fsType = "tmpfs"; @@ -50,11 +43,6 @@ type = "luks"; name = "encrypted"; settings = { - keyFile = "/dev/disk/by-label/NIXOS_DSC"; # The keyfile is stored on a USB stick - # The maximum size of the keyfile is 8192 KiB - # type `cryptsetup --help` to see the compiled-in key and passphrase maximum sizes - keyFileSize = 512 * 64; # match the `bs * count` of the `dd` command - keyFileOffset = 512 * 128; # match the `bs * skip` of the `dd` command # fallbackToPassword = true; allowDiscards = true; }; diff --git a/hosts/k8s/kubevirt-shoryu/default.nix b/hosts/k8s/kubevirt-shoryu/default.nix index 0846b70d..d9e09c53 100644 --- a/hosts/k8s/kubevirt-shoryu/default.nix +++ b/hosts/k8s/kubevirt-shoryu/default.nix @@ -17,7 +17,7 @@ let k3sModule = mylib.genK3sServerModule { inherit pkgs; kubeconfigFile = "/home/${myvars.username}/.kube/config"; - tokenFile = "/run/media/nixos_k3s/kubevirt-k3s-token"; + tokenFile = "/persistent/kubevirt-k3s-token"; # the first node in the cluster should be the one to initialize the cluster clusterInit = true; # use my own domain & kube-vip's virtual IP for the API server diff --git a/hosts/k8s/kubevirt-shushou/default.nix b/hosts/k8s/kubevirt-shushou/default.nix index 295f176f..2bde1c1e 100644 --- a/hosts/k8s/kubevirt-shushou/default.nix +++ b/hosts/k8s/kubevirt-shushou/default.nix @@ -16,7 +16,7 @@ let k3sModule = mylib.genK3sServerModule { inherit pkgs; kubeconfigFile = "/home/${myvars.username}/.kube/config"; - tokenFile = "/run/media/nixos_k3s/kubevirt-k3s-token"; + tokenFile = "/persistent/kubevirt-k3s-token"; # use my own domain & kube-vip's virtual IP for the API server # so that the API server can always be accessed even if some nodes are down masterHost = "kubevirt-cluster-1.writefor.fun"; diff --git a/hosts/k8s/kubevirt-youko/default.nix b/hosts/k8s/kubevirt-youko/default.nix index 6cea80a9..1e49bf14 100644 --- a/hosts/k8s/kubevirt-youko/default.nix +++ b/hosts/k8s/kubevirt-youko/default.nix @@ -16,7 +16,7 @@ let k3sModule = mylib.genK3sServerModule { inherit pkgs; kubeconfigFile = "/home/${myvars.username}/.kube/config"; - tokenFile = "/run/media/nixos_k3s/kubevirt-k3s-token"; + tokenFile = "/persistent/kubevirt-k3s-token"; # use my own domain & kube-vip's virtual IP for the API server # so that the API server can always be accessed even if some nodes are down masterHost = "kubevirt-cluster-1.writefor.fun"; diff --git a/lib/genKubeVirtHostModule.nix b/lib/genKubeVirtHostModule.nix index 12d622b2..2f80a2eb 100644 --- a/lib/genKubeVirtHostModule.nix +++ b/lib/genKubeVirtHostModule.nix @@ -3,11 +3,13 @@ hostName, networking, ... -}: let +}: +let inherit (networking) proxyGateway proxyGateway6 nameservers; inherit (networking.hostsAddr.${hostName}) iface ipv4; ipv4WithMask = "${ipv4}/24"; -in { +in +{ # supported file systems, so we can mount any removable disks with these filesystems boot.supportedFilesystems = [ "ext4" @@ -96,7 +98,7 @@ in { ovsbr1 = { # Attach the interfaces to OVS bridge # This interface should not used by the host itself! - interfaces.${iface} = {}; + interfaces.${iface} = { }; }; }; @@ -105,9 +107,9 @@ in { # Set the host's address on the OVS bridge interface instead of the physical interface! systemd.network.networks = { "10-ovsbr1" = { - matchConfig.Name = ["ovsbr1"]; + matchConfig.Name = [ "ovsbr1" ]; networkConfig = { - Address = [ipv4WithMask]; + Address = [ ipv4WithMask ]; DNS = nameservers; DHCP = "ipv6"; # enable DHCPv6 only, so we can get a GUA. IPv6AcceptRA = true; # for Stateless IPv6 Autoconfiguraton (SLAAC) @@ -127,7 +129,7 @@ in { linkConfig.RequiredForOnline = "routable"; }; "20-${iface}" = { - matchConfig.Name = [iface]; + matchConfig.Name = [ iface ]; networkConfig.LinkLocalAddressing = "no"; # tell networkd ignore this interface. # it's managed by openvswitch diff --git a/modules/nixos/base/zram.nix b/modules/nixos/base/zram.nix index d831887a..80ce1648 100644 --- a/modules/nixos/base/zram.nix +++ b/modules/nixos/base/zram.nix @@ -1,3 +1,4 @@ +{ lib, ... }: { # Enable in-memory compressed devices and swap space provided by the zram kernel module. # By enable this, we can store more data in memory instead of fallback to disk-based swap devices directly, @@ -7,37 +8,37 @@ zramSwap = { enable = true; # one of "lzo", "lz4", "zstd" - algorithm = "zstd"; + algorithm = lib.mkDefault "zstd"; # Priority of the zram swap devices. # It should be a number higher than the priority of your disk-based swap devices # (so that the system will fill the zram swap devices before falling back to disk swap). - priority = 100; + priority = lib.mkDefault 100; # Maximum total amount of memory that can be stored in the zram swap devices (as a percentage of your total memory). # Defaults to 1/2 of your total RAM. Run zramctl to check how good memory is compressed. # This doesn’t define how much memory will be used by the zram swap devices. - memoryPercent = 50; + memoryPercent = lib.mkDefault 50; }; # Optimizing swap on zram boot.kernel.sysctl = { # vm.swappiness - Controls kernel preference for swapping (range: 0-200, default: 60) # For in-memory swap devices like zram/zswap, values above 100 are recommended. - "vm.swappiness" = 180; + "vm.swappiness" = lib.mkDefault 180; # vm.watermark_boost_factor - Controls aggressiveness of memory reclaim (default: 15000) # Setting to 0 disables watermark boost, preventing premature memory reclamation. # This allows fuller memory utilization before the kernel starts reclaiming pages. - "vm.watermark_boost_factor" = 0; + "vm.watermark_boost_factor" = lib.mkDefault 0; # vm.watermark_scale_factor - Controls kswapd wakeup frequency (range: 1-1000, default: 10) # A higher value triggers background memory reclamation earlier (at 12.5% memory pressure). # Value 125 means kswapd becomes active when free memory drops below 1/125 of total memory, # balancing memory more proactively to prevent sudden swap storms at high swappiness values. - "vm.watermark_scale_factor" = 125; + "vm.watermark_scale_factor" = lib.mkDefault 125; # vm.page-cluster - Controls swap readahead (range: 0-6, default: 3) # 0 means read only 1 page (2^0) at a time, disabling readahead. # For low-latency devices like zram, readahead hurts performance by fetching unnecessary data. - "vm.page-cluster" = 0; + "vm.page-cluster" = lib.mkDefault 0; }; } diff --git a/modules/nixos/desktop.nix b/modules/nixos/desktop.nix index 11fc5a38..d1305c86 100644 --- a/modules/nixos/desktop.nix +++ b/modules/nixos/desktop.nix @@ -39,8 +39,8 @@ in user = myvars.username; # .wayland-session is a script generated by home-manager, which links to the current wayland compositor(sway/hyprland or others). # with such a vendor-no-locking script, we can switch to another wayland compositor without modifying greetd's config here. - command = "$HOME/.wayland-session"; # start a wayland session directly without a login manager - # command = "${pkgs.greetd.tuigreet}/bin/tuigreet --time --cmd $HOME/.wayland-session"; # start wayland session with a TUI login manager + command = "${pkgs.tuigreet}/bin/tuigreet --time --cmd $HOME/.wayland-session"; # start wayland session with a TUI login manager + # command = "$HOME/.wayland-session"; # start a wayland session directly without a login manager }; }; }; diff --git a/nixos-installer/README.md b/nixos-installer/README.md index d3f33eeb..f0fc986c 100644 --- a/nixos-installer/README.md +++ b/nixos-installer/README.md @@ -34,9 +34,17 @@ tmpfs; [preservation](https://github.com/nix-community/preservation) uses `/pers git clone https://github.com/ryan4yin/nix-config.git cd nix-config/nixos-installer -# WARNING: destroys all data on nvme1n1. Layout is mounted at /mnt by default. sudo su + +# encrypt the root partition with luks2 and argon2id, will prompt for a passphrase, which will be used to unlock the partition. +# WARNING: destroys all data on nvme1n1. Layout is mounted at /mnt by default. nix run github:nix-community/disko -- --mode destroy,format,mount ../hosts/idols-ai/disko-fs.nix + +# Mount only (e.g. after first format, without wiping): +# nix run github:nix-community/disko -- --mode mount ../hosts/idols-ai/disko-fs.nix + +# setup the automatic unlock via the tpm2 chip +systemd-cryptenroll --tpm2-device=auto --tpm2-pcrs=0+7 /dev/ ``` ### 2. Install NixOS diff --git a/nixos-installer/README.shoukei.md b/nixos-installer/README.shoukei.md index ffc582eb..4a6a5555 100755 --- a/nixos-installer/README.shoukei.md +++ b/nixos-installer/README.shoukei.md @@ -134,6 +134,9 @@ cryptsetup luksOpen /dev/nvme0n1p6 crypted-nixos # show disk status lsblk + +# setup the automatic unlock via the tpm2 chip +systemd-cryptenroll --tpm2-device=auto --tpm2-pcrs=0+7 /dev/nvme0n1p6 ``` Formatting the root partition: diff --git a/vars/networking.nix b/vars/networking.nix index f43faf86..88de3a11 100644 --- a/vars/networking.nix +++ b/vars/networking.nix @@ -38,7 +38,7 @@ rec { # ============================================ ai = { # Desktop PC - iface = "enp4s0"; + iface = "enp5s0"; ipv4 = "192.168.5.100"; ipv6 = "fe80::10"; # Link-local Address };