diff --git a/hosts/k8s/disko-config/README.md b/hosts/k8s/disko-config/README.md index 8fd2ce9e..acc95e33 100644 --- a/hosts/k8s/disko-config/README.md +++ b/hosts/k8s/disko-config/README.md @@ -4,15 +4,16 @@ Generate LUKS keyfile to encrypt the root partition, it's used by disko. ```bash # partition the usb stick -parted /dev/sdb -- mklabel gpt -parted /dev/sdb -- mkpart primary 2M 512MB -parted /dev/sdb -- mkpart primary 512MB 1024MB -mkfs.fat -F 32 -n NIXOS_DSC /dev/sdb1 -mkfs.fat -F 32 -n NIXOS_K3S /dev/sdb2 +DEV=/dev/sdX +parted $DEV -- mklabel gpt +parted $DEV -- mkpart primary 2M 512MB +parted $DEV -- mkpart primary 512MB 1024MB +mkfs.fat -F 32 -n NIXOS_DSC ${DEV}1 +mkfs.fat -F 32 -n NIXOS_K3S ${DEV}2 # Generate a keyfile from the true random number generator KEYFILE=./kubevirt-luks-keyfile -dd bs=8192 count=4 iflag=fullblock if=/dev/random of=$KEYFILE +dd bs=512 count=64 iflag=fullblock if=/dev/random of=$KEYFILE # generate token for k3s K3S_TOKEN_FILE=./kubevirt-k3s-token @@ -20,12 +21,25 @@ K3S_TOKEN=$(grep -ao '[A-Za-z0-9]' < /dev/random | head -64 | tr -d '\n' ; echo echo $K3S_TOKEN > $K3S_TOKEN_FILE # copy the keyfile and token to the usb stick - KEYFILE=./kubevirt-luks-keyfile DEVICE=/dev/disk/by-label/NIXOS_DSC -dd bs=8192 count=4 iflag=fullblock if=$KEYFILE of=$DEVICE +# seek=128 skip N obs-sized output blocks to avoid overwriting the filesystem header +dd bs=512 count=64 iflag=fullblock seek=128 if=$KEYFILE of=$DEVICE K3S_TOKEN_FILE=./kubevirt-k3s-token USB_PATH=/run/media/ryan/NIXOS_K3S cp $K3S_TOKEN_FILE $USB_PATH ``` + +### 2. Partition the SSD & install NixOS via disko + +```bash +# enter an shell with git/vim/ssh-agent/gnumake available +nix-shell -p git vim gnumake +# clone this repository +git clone https://github.com/ryan4yin/nix-config.git + +cd nix-config +sudo nix run --experimental-features "nix-command flakes" 'github:nix-community/disko#disko-install' -- \ + --write-efi-boot-entries --flake .#kubevirt-shoryu --disk main /dev/nvme0n1 +``` diff --git a/hosts/k8s/disko-config/kubevirt-disko-fs.nix b/hosts/k8s/disko-config/kubevirt-disko-fs.nix index d2bc5f40..8d8c385b 100644 --- a/hosts/k8s/disko-config/kubevirt-disko-fs.nix +++ b/hosts/k8s/disko-config/kubevirt-disko-fs.nix @@ -1,4 +1,7 @@ { + # required by impermanence + fileSystems."/persistent".neededForBoot = true; + # contains the k3s's token fileSystems."/run/media/nixos_k3s" = { device = "/dev/disk/by-label/NIXOS_K3S"; @@ -7,90 +10,92 @@ }; disko.devices = { - disk = { - sda = { - type = "disk"; - device = "/dev/nvme0n1"; - content = { - type = "gpt"; - partitions = { - # The EFI & Boot partition - ESP = { - size = "630M"; - type = "EF00"; - content = { - type = "filesystem"; - format = "vfat"; - mountpoint = "/boot/efi"; - mountOptions = [ - "defaults" - ]; - }; + nodev."/" = { + fsType = "tmpfs"; + mountOptions = [ + "size=4G" + "defaults" + # set mode to 755, otherwise systemd will set it to 777, which cause problems. + # relatime: Update inode access times relative to modify or change time. + "mode=755" + ]; + }; + + disk.main = { + type = "disk"; + # When using disko-install, we will overwrite this value from the commandline + device = "/dev/nvme0n1"; # The device to partition + content = { + type = "gpt"; + partitions = { + # The EFI & Boot partition + ESP = { + size = "630M"; + type = "EF00"; + content = { + type = "filesystem"; + format = "vfat"; + mountpoint = "/boot"; + mountOptions = [ + "defaults" + ]; }; - # The root partition - luks = { - size = "100%"; + }; + # The root partition + luks = { + size = "100%"; + content = { + type = "luks"; + name = "encrypted"; + settings = { + keyFile = "/dev/disk/by-label/NIXOS_DSC"; # The keyfile is stored on a USB stick + # The maximum size of the keyfile is 8192 bytes + keyFileSize = 512 * 64; # match the `bs * count` of the `dd` command + keyFileOffset = 512 * 128; # match the `bs * skip` of the `dd` command + fallbackToPassword = true; + allowDiscards = true; + }; + # Whether to add a boot.initrd.luks.devices entry for the specified disk. + initrdUnlock = true; + + # encrypt the root partition with luks2 and argon2id, will prompt for a passphrase, which will be used to unlock the partition. + # cryptsetup luksFormat + extraFormatArgs = [ + "--type luks2" + "--cipher aes-xts-plain64" + "--hash sha512" + "--iter-time 5000" + "--key-size 256" + "--pbkdf argon2id" + # use true random data from /dev/random, will block until enough entropy is available + "--use-random" + ]; + extraOpenArgs = [ + "--timeout 10" + ]; content = { - type = "luks"; - name = "crypted"; - settings = { - keyFile = "/dev/disk/by-label/NIXOS_DSC"; # The keyfile is stored on a USB stick - keyFileSize = 8192 * 4; # The maxium size of the keyfile is 8192 bytes - keyFileOffset = 0; - fallbackToPassword = true; - allowDiscards = true; - }; - # Whether to add a boot.initrd.luks.devices entry for the specified disk. - initrdUnlock = true; - - # encrypt the root partition with luks2 and argon2id, will prompt for a passphrase, which will be used to unlock the partition. - # cryptsetup luksFormat - extraFormatArgs = [ - "--type luks2" - "--cipher aes-xts-plain64" - "--hash sha512" - "--iter-time 5000" - "--key-size 256" - "--pbkdf argon2id" - # use true random data from /dev/random, will block until enough entropy is available - "--use-random" - ]; - extraOpenArgs = [ - "--timeout 10" - ]; - content = { - type = "btrfs"; - extraArgs = ["-f"]; - subvolumes = { - "@root" = { - mountpoint = "/"; - mountOptions = ["compress-force=zstd:1" "noatime"]; - }; - "@home" = { - mountpoint = "/home"; - mountOptions = ["compress-force=zstd:1"]; - }; - "@lib" = { - mountpoint = "/var/lib"; - mountOptions = ["compress-force=zstd:1"]; - }; - - "@nix" = { - mountpoint = "/nix"; - mountOptions = ["compress-force=zstd:1" "noatime"]; - }; - "@tmp" = { - mountpoint = "/tmp"; - mountOptions = ["compress-force=zstd:1" "noatime"]; - }; - "@snapshots" = { - mountpoint = "/snapshots"; - mountOptions = ["compress-force=zstd:1" "noatime"]; - }; - "@swap" = { - mountpoint = "/swap"; - swap.swapfile.size = "8192M"; - }; + type = "btrfs"; + extraArgs = ["-f"]; + subvolumes = { + "@nix" = { + mountpoint = "/nix"; + mountOptions = ["compress-force=zstd:1" "noatime"]; + }; + "@persistent" = { + mountpoint = "/persistent"; + mountOptions = ["compress-force=zstd:1" "noatime"]; + }; + "@tmp" = { + mountpoint = "/tmp"; + mountOptions = ["compress-force=zstd:1" "noatime"]; + }; + "@snapshots" = { + mountpoint = "/snapshots"; + mountOptions = ["compress-force=zstd:1" "noatime"]; + }; + "@swap" = { + mountpoint = "/swap"; + swap.swapfile.size = "16384M"; }; }; }; diff --git a/hosts/k8s/kubevirt-shoryu/default.nix b/hosts/k8s/kubevirt-shoryu/default.nix index d108f8e2..9f45746b 100644 --- a/hosts/k8s/kubevirt-shoryu/default.nix +++ b/hosts/k8s/kubevirt-shoryu/default.nix @@ -8,7 +8,7 @@ }: let # MoreFine - S500Plus hostName = "kubevirt-shoryu"; # Define your hostname. - + coreModule = mylib.genKubeVirtCoreModule { inherit pkgs hostName; inherit (myvars) networking; @@ -16,7 +16,7 @@ k3sModule = mylib.genK3sServerModule { inherit pkgs; kubeconfigFile = "/home/${myvars.username}/.kube/config"; - tokenFile = config.age.secrets."k3s-prod-1-token".path; + tokenFile = "/run/media/nixos_k3s/kubevirt-k3s-token"; # the first node in the cluster should be the one to initialize the cluster clusterInit = true; }; @@ -26,6 +26,8 @@ in { ++ [ disko.nixosModules.default ../disko-config/kubevirt-disko-fs.nix + ./hardware-configuration.nix + ./impermanence.nix coreModule k3sModule ]; diff --git a/hosts/k8s/kubevirt-shoryu/hardware-configuration.nix b/hosts/k8s/kubevirt-shoryu/hardware-configuration.nix new file mode 100644 index 00000000..6302c527 --- /dev/null +++ b/hosts/k8s/kubevirt-shoryu/hardware-configuration.nix @@ -0,0 +1,49 @@ +{ + config, + lib, + pkgs, + modulesPath, + ... +}: { + imports = [ + (modulesPath + "/installer/scan/not-detected.nix") + ]; + + boot.loader = { + # depending on how you configured your disk mounts, change this to /boot or /boot/efi. + efi.efiSysMountPoint = "/boot/"; + efi.canTouchEfiVariables = true; + systemd-boot.enable = true; + }; + # clear /tmp on boot to get a stateless /tmp directory. + boot.tmp.cleanOnBoot = true; + + boot.initrd.availableKernelModules = ["xhci_pci" "ahci" "nvme" "usbhid" "usb_storage" "sd_mod"]; + boot.kernelModules = ["kvm-amd" "vfio-pci"]; + boot.extraModprobeConfig = "options kvm_amd nested=1"; # for amd cpu + + # Enable binfmt emulation of aarch64-linux, this is required for cross compilation. + boot.binfmt.emulatedSystems = ["aarch64-linux" "riscv64-linux"]; + # supported file systems, so we can mount any removable disks with these filesystems + boot.supportedFilesystems = [ + "ext4" + "btrfs" + "xfs" + "ntfs" + "fat" + "vfat" + "cifs" # mount windows share + ]; + + # Enables DHCP on each ethernet and wireless interface. In case of scripted networking + # (the default) this is the recommended approach. When using systemd-networkd it's + # still possible to use this option, but it's recommended to use it in conjunction + # with explicit per-interface declarations with `networking.interfaces..useDHCP`. + networking.useDHCP = lib.mkDefault true; + # networking.interfaces.enp5s0.useDHCP = lib.mkDefault true; + # networking.interfaces.wlo1.useDHCP = lib.mkDefault true; + + nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux"; + powerManagement.cpuFreqGovernor = lib.mkDefault "powersave"; + hardware.cpu.intel.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware; +} diff --git a/hosts/k8s/kubevirt-shoryu/impermanence.nix b/hosts/k8s/kubevirt-shoryu/impermanence.nix new file mode 100644 index 00000000..4cb6aaa2 --- /dev/null +++ b/hosts/k8s/kubevirt-shoryu/impermanence.nix @@ -0,0 +1,53 @@ +{ + impermanence, + pkgs, + ... +}: { + imports = [ + impermanence.nixosModules.impermanence + ]; + + environment.systemPackages = [ + # `sudo ncdu -x /` + pkgs.ncdu + ]; + + # NOTE: impermanence only mounts the directory/file list below to /persistent + # If the directory/file already exists in the root filesystem, you should + # move those files/directories to /persistent first! + environment.persistence."/persistent" = { + # sets the mount option x-gvfs-hide on all the bind mounts + # to hide them from the file manager + hideMounts = true; + directories = [ + "/etc/NetworkManager/system-connections" + "/etc/ssh" + "/etc/nix/inputs" + "/etc/secureboot" # lanzaboote - secure boot + # my secrets + "/etc/agenix/" + + "/var/log" + "/var/lib" + + # k3s related + "/etc/iscsi" + "/etc/rancher" + ]; + files = [ + "/etc/machine-id" + ]; + + # the following directories will be passed to /persistent/home/$USER + users.ryan = { + directories = [ + "codes" + "nix-config" + "tmp" + ]; + files = [ + ".config/nushell/history.txt" + ]; + }; + }; +} diff --git a/hosts/k8s/kubevirt-shushou/default.nix b/hosts/k8s/kubevirt-shushou/default.nix index 2de3c6a1..2b2e27c4 100644 --- a/hosts/k8s/kubevirt-shushou/default.nix +++ b/hosts/k8s/kubevirt-shushou/default.nix @@ -16,7 +16,7 @@ k3sModule = mylib.genK3sServerModule { inherit pkgs; kubeconfigFile = "/home/${myvars.username}/.kube/config"; - tokenFile = config.age.secrets."k3s-prod-1-token".path; + tokenFile = "/run/media/nixos_k3s/kubevirt-k3s-token"; serverIp = myvars.networking.hostsAddr.${k3sServerName}.ipv4; }; in { @@ -25,6 +25,8 @@ in { ++ [ disko.nixosModules.default ../disko-config/kubevirt-disko-fs.nix + ../kubevirt-shoryu/hardware-configuration.nix + ../kubevirt-shoryu/impermanence.nix coreModule k3sModule ]; diff --git a/hosts/k8s/kubevirt-youko/default.nix b/hosts/k8s/kubevirt-youko/default.nix index 137c391a..e207f439 100644 --- a/hosts/k8s/kubevirt-youko/default.nix +++ b/hosts/k8s/kubevirt-youko/default.nix @@ -16,7 +16,7 @@ k3sModule = mylib.genK3sServerModule { inherit pkgs; kubeconfigFile = "/home/${myvars.username}/.kube/config"; - tokenFile = config.age.secrets."k3s-prod-1-token".path; + tokenFile = "/run/media/nixos_k3s/kubevirt-k3s-token"; serverIp = myvars.networking.hostsAddr.${k3sServerName}.ipv4; }; in { @@ -25,6 +25,8 @@ in { ++ [ disko.nixosModules.default ../disko-config/kubevirt-disko-fs.nix + ../kubevirt-shoryu/hardware-configuration.nix + ../kubevirt-shoryu/impermanence.nix coreModule k3sModule ]; diff --git a/lib/genK3sAgentModule.nix b/lib/genK3sAgentModule.nix index fabcf12d..f536aca1 100644 --- a/lib/genK3sAgentModule.nix +++ b/lib/genK3sAgentModule.nix @@ -14,8 +14,12 @@ in { role = "agent"; serverAddr = "https://${serverIp}:6443"; # https://docs.k3s.io/cli/agent - extraFlags = - " --node-label=node-type=worker" - + " --data-dir /var/lib/rancher/k3s"; + extraFlags = let + flagList = [ + "--node-label=node-type=worker" + "--data-dir /var/lib/rancher/k3s" + ]; + in + pkgs.lib.concatStringsSep " " flagList; }; } diff --git a/lib/genK3sServerModule.nix b/lib/genK3sServerModule.nix index e88bd85a..655e5279 100644 --- a/lib/genK3sServerModule.nix +++ b/lib/genK3sServerModule.nix @@ -8,6 +8,7 @@ # and other servers must connect to it using `serverAddr`. serverIp ? null, clusterInit ? (serverIp == null), + addTaints ? false, ... }: let package = pkgs.k3s_1_29; @@ -18,6 +19,8 @@ in { kubectl istioctl kubernetes-helm + cilium-cli + fluxcd skopeo dive # explore docker layers @@ -33,20 +36,33 @@ in { role = "server"; # https://docs.k3s.io/cli/server - extraFlags = - " --write-kubeconfig ${kubeconfigFile}" - + " --write-kubeconfig-mode 644" - + " --service-node-port-range 80-32767" - + " --kube-apiserver-arg='--allow-privileged=true'" # required by kubevirt - + " --node-taint=CriticalAddonsOnly=true:NoExecute" # prevent workloads from running on the master - + " --data-dir /var/lib/rancher/k3s" - + " --etcd-expose-metrics true" - + " --etcd-snapshot-schedule-cron '0 */12 * * *'" - # disable some features we don't need - + " --disable-helm-controller" # we use fluxcd instead - + " --disable=traefik" # deploy our own ingress controller instead - + " --disable=servicelb" # we use kube-vip instead - + " --flannel-backend=none" # we use cilium instead - + " --disable-network-policy"; + extraFlags = let + flagList = + [ + "--write-kubeconfig ${kubeconfigFile}" + "--write-kubeconfig-mode=644" + "--service-node-port-range=80-32767" + "--kube-apiserver-arg='--allow-privileged=true'" # required by kubevirt + "--data-dir /var/lib/rancher/k3s" + "--etcd-expose-metrics=true" + "--etcd-snapshot-schedule-cron='0 */12 * * *'" + # disable some features we don't need + "--disable-helm-controller" # we use fluxcd instead + "--disable=traefik" # deploy our own ingress controller instead + "--disable=servicelb" # we use kube-vip instead + "--flannel-backend=none" # we use cilium instead + "--disable-network-policy" + ] + # prevent workloads from running on the master + ++ (pkgs.lib.optionals addTaints ["--node-taint=CriticalAddonsOnly=true:NoExecute"]); + in + pkgs.lib.concatStringsSep " " flagList; }; + + # create symlinks to link k3s's cni directory to the one used by almost all CNI plugins + systemd.tmpfiles.rules = [ + "L+ /opt/cni/bin - - - - /var/lib/rancher/k3s/data/current/bin" + # seems like k3s's containerd will create /etc/cni/net.d, so we don't need to create a symlink for it + # "L+ /etc/cni/net.d - - - - /var/lib/rancher/k3s/agent/etc/cni/net.d" + ]; } diff --git a/modules/nixos/server/server.nix b/modules/nixos/server/server.nix index f3d25d78..a1d88467 100644 --- a/modules/nixos/server/server.nix +++ b/modules/nixos/server/server.nix @@ -3,4 +3,5 @@ ../base ../../base.nix ]; + boot.loader.timeout = 3; # wait for 3 seconds to select the boot entry }