Merge pull request #109 from ryan4yin/kubevirt

feat: add configs for kubevirt-lab-1

hosts/k8s/disko-config/README.md

@@ -4,15 +4,16 @@ Generate LUKS keyfile to encrypt the root partition, it's used by disko.
 
 ```bash
 # partition the usb stick
-parted /dev/sdb -- mklabel gpt
+DEV=/dev/sdX
-parted /dev/sdb -- mkpart primary 2M 512MB
+parted $DEV -- mklabel gpt
-parted /dev/sdb -- mkpart primary 512MB 1024MB
+parted $DEV -- mkpart primary 2M 512MB
-mkfs.fat -F 32 -n NIXOS_DSC /dev/sdb1
+parted $DEV -- mkpart primary 512MB 1024MB
-mkfs.fat -F 32 -n NIXOS_K3S /dev/sdb2
+mkfs.fat -F 32 -n NIXOS_DSC ${DEV}1
+mkfs.fat -F 32 -n NIXOS_K3S ${DEV}2
 
 # Generate a keyfile from the true random number generator
 KEYFILE=./kubevirt-luks-keyfile
-dd bs=8192 count=4 iflag=fullblock if=/dev/random of=$KEYFILE
+dd bs=512 count=64 iflag=fullblock if=/dev/random of=$KEYFILE
 
 # generate token for k3s
 K3S_TOKEN_FILE=./kubevirt-k3s-token
@@ -20,12 +21,25 @@ K3S_TOKEN=$(grep -ao '[A-Za-z0-9]' < /dev/random | head -64 | tr -d '\n' ; echo
 echo $K3S_TOKEN > $K3S_TOKEN_FILE
 
 # copy the keyfile and token to the usb stick
-
 KEYFILE=./kubevirt-luks-keyfile
 DEVICE=/dev/disk/by-label/NIXOS_DSC
-dd bs=8192 count=4 iflag=fullblock if=$KEYFILE of=$DEVICE
+# seek=128 skip N obs-sized output blocks to avoid overwriting the filesystem header
+dd bs=512 count=64 iflag=fullblock seek=128 if=$KEYFILE of=$DEVICE
 
 K3S_TOKEN_FILE=./kubevirt-k3s-token
 USB_PATH=/run/media/ryan/NIXOS_K3S
 cp $K3S_TOKEN_FILE $USB_PATH
 ```
+
+### 2. Partition the SSD & install NixOS via disko
+
+```bash
+# enter an shell with git/vim/ssh-agent/gnumake available
+nix-shell -p git vim gnumake
+# clone this repository
+git clone https://github.com/ryan4yin/nix-config.git
+
+cd nix-config
+sudo nix run --experimental-features "nix-command flakes" 'github:nix-community/disko#disko-install' -- \
+  --write-efi-boot-entries --flake .#kubevirt-shoryu --disk main /dev/nvme0n1
+```

hosts/k8s/disko-config/kubevirt-disko-fs.nix

@@ -1,4 +1,7 @@
 {
+  # required by impermanence
+  fileSystems."/persistent".neededForBoot = true;
+
   # contains the k3s's token
   fileSystems."/run/media/nixos_k3s" = {
     device = "/dev/disk/by-label/NIXOS_K3S";
@@ -7,90 +10,92 @@
   };
 
   disko.devices = {
-    disk = {
+    nodev."/" = {
-      sda = {
+      fsType = "tmpfs";
-        type = "disk";
+      mountOptions = [
-        device = "/dev/nvme0n1";
+        "size=4G"
-        content = {
+        "defaults"
-          type = "gpt";
+        # set mode to 755, otherwise systemd will set it to 777, which cause problems.
-          partitions = {
+        # relatime: Update inode access times relative to modify or change time.
-            # The EFI & Boot partition
+        "mode=755"
-            ESP = {
+      ];
-              size = "630M";
+    };
-              type = "EF00";
+
-              content = {
+    disk.main = {
-                type = "filesystem";
+      type = "disk";
-                format = "vfat";
+      # When using disko-install, we will overwrite this value from the commandline
-                mountpoint = "/boot/efi";
+      device = "/dev/nvme0n1"; # The device to partition
-                mountOptions = [
+      content = {
-                  "defaults"
+        type = "gpt";
-                ];
+        partitions = {
-              };
+          # The EFI & Boot partition
+          ESP = {
+            size = "630M";
+            type = "EF00";
+            content = {
+              type = "filesystem";
+              format = "vfat";
+              mountpoint = "/boot";
+              mountOptions = [
+                "defaults"
+              ];
             };
-            # The root partition
+          };
-            luks = {
+          # The root partition
-              size = "100%";
+          luks = {
+            size = "100%";
+            content = {
+              type = "luks";
+              name = "encrypted";
+              settings = {
+                keyFile = "/dev/disk/by-label/NIXOS_DSC"; # The keyfile is stored on a USB stick
+                # The maximum size of the keyfile is 8192 bytes
+                keyFileSize = 512 * 64; # match the `bs * count` of the `dd` command
+                keyFileOffset = 512 * 128; # match the `bs * skip` of the `dd` command
+                fallbackToPassword = true;
+                allowDiscards = true;
+              };
+              # Whether to add a boot.initrd.luks.devices entry for the specified disk.
+              initrdUnlock = true;
+
+              # encrypt the root partition with luks2 and argon2id, will prompt for a passphrase, which will be used to unlock the partition.
+              # cryptsetup luksFormat
+              extraFormatArgs = [
+                "--type luks2"
+                "--cipher aes-xts-plain64"
+                "--hash sha512"
+                "--iter-time 5000"
+                "--key-size 256"
+                "--pbkdf argon2id"
+                # use true random data from /dev/random, will block until enough entropy is available
+                "--use-random"
+              ];
+              extraOpenArgs = [
+                "--timeout 10"
+              ];
               content = {
-                type = "luks";
+                type = "btrfs";
-                name = "crypted";
+                extraArgs = ["-f"];
-                settings = {
+                subvolumes = {
-                  keyFile = "/dev/disk/by-label/NIXOS_DSC"; # The keyfile is stored on a USB stick
+                  "@nix" = {
-                  keyFileSize = 8192 * 4; # The maxium size of the keyfile is 8192 bytes
+                    mountpoint = "/nix";
-                  keyFileOffset = 0;
+                    mountOptions = ["compress-force=zstd:1" "noatime"];
-                  fallbackToPassword = true;
+                  };
-                  allowDiscards = true;
+                  "@persistent" = {
-                };
+                    mountpoint = "/persistent";
-                # Whether to add a boot.initrd.luks.devices entry for the specified disk.
+                    mountOptions = ["compress-force=zstd:1" "noatime"];
-                initrdUnlock = true;
+                  };
-
+                  "@tmp" = {
-                # encrypt the root partition with luks2 and argon2id, will prompt for a passphrase, which will be used to unlock the partition.
+                    mountpoint = "/tmp";
-                # cryptsetup luksFormat
+                    mountOptions = ["compress-force=zstd:1" "noatime"];
-                extraFormatArgs = [
+                  };
-                  "--type luks2"
+                  "@snapshots" = {
-                  "--cipher aes-xts-plain64"
+                    mountpoint = "/snapshots";
-                  "--hash sha512"
+                    mountOptions = ["compress-force=zstd:1" "noatime"];
-                  "--iter-time 5000"
+                  };
-                  "--key-size 256"
+                  "@swap" = {
-                  "--pbkdf argon2id"
+                    mountpoint = "/swap";
-                  # use true random data from /dev/random, will block until enough entropy is available
+                    swap.swapfile.size = "16384M";
-                  "--use-random"
-                ];
-                extraOpenArgs = [
-                  "--timeout 10"
-                ];
-                content = {
-                  type = "btrfs";
-                  extraArgs = ["-f"];
-                  subvolumes = {
-                    "@root" = {
-                      mountpoint = "/";
-                      mountOptions = ["compress-force=zstd:1" "noatime"];
-                    };
-                    "@home" = {
-                      mountpoint = "/home";
-                      mountOptions = ["compress-force=zstd:1"];
-                    };
-                    "@lib" = {
-                      mountpoint = "/var/lib";
-                      mountOptions = ["compress-force=zstd:1"];
-                    };
-
-                    "@nix" = {
-                      mountpoint = "/nix";
-                      mountOptions = ["compress-force=zstd:1" "noatime"];
-                    };
-                    "@tmp" = {
-                      mountpoint = "/tmp";
-                      mountOptions = ["compress-force=zstd:1" "noatime"];
-                    };
-                    "@snapshots" = {
-                      mountpoint = "/snapshots";
-                      mountOptions = ["compress-force=zstd:1" "noatime"];
-                    };
-                    "@swap" = {
-                      mountpoint = "/swap";
-                      swap.swapfile.size = "8192M";
-                    };
                   };
                 };
               };

hosts/k8s/kubevirt-shoryu/default.nix

@@ -8,7 +8,7 @@
 }: let
   # MoreFine - S500Plus
   hostName = "kubevirt-shoryu"; # Define your hostname.
-  
+
   coreModule = mylib.genKubeVirtCoreModule {
     inherit pkgs hostName;
     inherit (myvars) networking;
@@ -16,7 +16,7 @@
   k3sModule = mylib.genK3sServerModule {
     inherit pkgs;
     kubeconfigFile = "/home/${myvars.username}/.kube/config";
-    tokenFile = config.age.secrets."k3s-prod-1-token".path;
+    tokenFile = "/run/media/nixos_k3s/kubevirt-k3s-token";
     # the first node in the cluster should be the one to initialize the cluster
     clusterInit = true;
   };
@@ -26,6 +26,8 @@ in {
     ++ [
       disko.nixosModules.default
       ../disko-config/kubevirt-disko-fs.nix
+      ./hardware-configuration.nix
+      ./impermanence.nix
       coreModule
       k3sModule
     ];

hosts/k8s/kubevirt-shoryu/hardware-configuration.nix

@@ -0,0 +1,49 @@
+{
+  config,
+  lib,
+  pkgs,
+  modulesPath,
+  ...
+}: {
+  imports = [
+    (modulesPath + "/installer/scan/not-detected.nix")
+  ];
+
+  boot.loader = {
+    # depending on how you configured your disk mounts, change this to /boot or /boot/efi.
+    efi.efiSysMountPoint = "/boot/";
+    efi.canTouchEfiVariables = true;
+    systemd-boot.enable = true;
+  };
+  # clear /tmp on boot to get a stateless /tmp directory.
+  boot.tmp.cleanOnBoot = true;
+
+  boot.initrd.availableKernelModules = ["xhci_pci" "ahci" "nvme" "usbhid" "usb_storage" "sd_mod"];
+  boot.kernelModules = ["kvm-amd" "vfio-pci"];
+  boot.extraModprobeConfig = "options kvm_amd nested=1"; # for amd cpu
+
+  # Enable binfmt emulation of aarch64-linux, this is required for cross compilation.
+  boot.binfmt.emulatedSystems = ["aarch64-linux" "riscv64-linux"];
+  # supported file systems, so we can mount any removable disks with these filesystems
+  boot.supportedFilesystems = [
+    "ext4"
+    "btrfs"
+    "xfs"
+    "ntfs"
+    "fat"
+    "vfat"
+    "cifs" # mount windows share
+  ];
+
+  # Enables DHCP on each ethernet and wireless interface. In case of scripted networking
+  # (the default) this is the recommended approach. When using systemd-networkd it's
+  # still possible to use this option, but it's recommended to use it in conjunction
+  # with explicit per-interface declarations with `networking.interfaces.<interface>.useDHCP`.
+  networking.useDHCP = lib.mkDefault true;
+  # networking.interfaces.enp5s0.useDHCP = lib.mkDefault true;
+  # networking.interfaces.wlo1.useDHCP = lib.mkDefault true;
+
+  nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux";
+  powerManagement.cpuFreqGovernor = lib.mkDefault "powersave";
+  hardware.cpu.intel.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware;
+}

hosts/k8s/kubevirt-shoryu/impermanence.nix

@@ -0,0 +1,53 @@
+{
+  impermanence,
+  pkgs,
+  ...
+}: {
+  imports = [
+    impermanence.nixosModules.impermanence
+  ];
+
+  environment.systemPackages = [
+    # `sudo ncdu -x /`
+    pkgs.ncdu
+  ];
+
+  # NOTE: impermanence only mounts the directory/file list below to /persistent
+  # If the directory/file already exists in the root filesystem, you should
+  # move those files/directories to /persistent first!
+  environment.persistence."/persistent" = {
+    # sets the mount option x-gvfs-hide on all the bind mounts
+    # to hide them from the file manager
+    hideMounts = true;
+    directories = [
+      "/etc/NetworkManager/system-connections"
+      "/etc/ssh"
+      "/etc/nix/inputs"
+      "/etc/secureboot" # lanzaboote - secure boot
+      # my secrets
+      "/etc/agenix/"
+
+      "/var/log"
+      "/var/lib"
+
+      # k3s related
+      "/etc/iscsi"
+      "/etc/rancher"
+    ];
+    files = [
+      "/etc/machine-id"
+    ];
+
+    # the following directories will be passed to /persistent/home/$USER
+    users.ryan = {
+      directories = [
+        "codes"
+        "nix-config"
+        "tmp"
+      ];
+      files = [
+        ".config/nushell/history.txt"
+      ];
+    };
+  };
+}

hosts/k8s/kubevirt-shushou/default.nix

@@ -16,7 +16,7 @@
   k3sModule = mylib.genK3sServerModule {
     inherit pkgs;
     kubeconfigFile = "/home/${myvars.username}/.kube/config";
-    tokenFile = config.age.secrets."k3s-prod-1-token".path;
+    tokenFile = "/run/media/nixos_k3s/kubevirt-k3s-token";
     serverIp = myvars.networking.hostsAddr.${k3sServerName}.ipv4;
   };
 in {
@@ -25,6 +25,8 @@ in {
     ++ [
       disko.nixosModules.default
       ../disko-config/kubevirt-disko-fs.nix
+      ../kubevirt-shoryu/hardware-configuration.nix
+      ../kubevirt-shoryu/impermanence.nix
       coreModule
       k3sModule
     ];

hosts/k8s/kubevirt-youko/default.nix

@@ -16,7 +16,7 @@
   k3sModule = mylib.genK3sServerModule {
     inherit pkgs;
     kubeconfigFile = "/home/${myvars.username}/.kube/config";
-    tokenFile = config.age.secrets."k3s-prod-1-token".path;
+    tokenFile = "/run/media/nixos_k3s/kubevirt-k3s-token";
     serverIp = myvars.networking.hostsAddr.${k3sServerName}.ipv4;
   };
 in {
@@ -25,6 +25,8 @@ in {
     ++ [
       disko.nixosModules.default
       ../disko-config/kubevirt-disko-fs.nix
+      ../kubevirt-shoryu/hardware-configuration.nix
+      ../kubevirt-shoryu/impermanence.nix
       coreModule
       k3sModule
     ];

lib/genK3sAgentModule.nix

@@ -14,8 +14,12 @@ in {
     role = "agent";
     serverAddr = "https://${serverIp}:6443";
     # https://docs.k3s.io/cli/agent
-    extraFlags =
+    extraFlags = let
-      " --node-label=node-type=worker"
+      flagList = [
-      + " --data-dir /var/lib/rancher/k3s";
+        "--node-label=node-type=worker"
+        "--data-dir /var/lib/rancher/k3s"
+      ];
+    in
+      pkgs.lib.concatStringsSep " " flagList;
   };
 }

lib/genK3sServerModule.nix

@@ -8,6 +8,7 @@
   # and other servers must connect to it using `serverAddr`.
   serverIp ? null,
   clusterInit ? (serverIp == null),
+  addTaints ? false,
   ...
 }: let
   package = pkgs.k3s_1_29;
@@ -18,6 +19,8 @@ in {
     kubectl
     istioctl
     kubernetes-helm
+    cilium-cli
+    fluxcd
 
     skopeo
     dive # explore docker layers
@@ -33,20 +36,33 @@ in {
 
     role = "server";
     # https://docs.k3s.io/cli/server
-    extraFlags =
+    extraFlags = let
-      " --write-kubeconfig ${kubeconfigFile}"
+      flagList =
-      + " --write-kubeconfig-mode 644"
+        [
-      + " --service-node-port-range 80-32767"
+          "--write-kubeconfig ${kubeconfigFile}"
-      + " --kube-apiserver-arg='--allow-privileged=true'" # required by kubevirt
+          "--write-kubeconfig-mode=644"
-      + " --node-taint=CriticalAddonsOnly=true:NoExecute" # prevent workloads from running on the master
+          "--service-node-port-range=80-32767"
-      + " --data-dir /var/lib/rancher/k3s"
+          "--kube-apiserver-arg='--allow-privileged=true'" # required by kubevirt
-      + " --etcd-expose-metrics true"
+          "--data-dir /var/lib/rancher/k3s"
-      + " --etcd-snapshot-schedule-cron '0 */12 * * *'"
+          "--etcd-expose-metrics=true"
-      # disable some features we don't need
+          "--etcd-snapshot-schedule-cron='0 */12 * * *'"
-      + " --disable-helm-controller" # we use fluxcd instead
+          # disable some features we don't need
-      + " --disable=traefik" # deploy our own ingress controller instead
+          "--disable-helm-controller" # we use fluxcd instead
-      + " --disable=servicelb" # we use kube-vip instead
+          "--disable=traefik" # deploy our own ingress controller instead
-      + " --flannel-backend=none" # we use cilium instead
+          "--disable=servicelb" # we use kube-vip instead
-      + " --disable-network-policy";
+          "--flannel-backend=none" # we use cilium instead
+          "--disable-network-policy"
+        ]
+        # prevent workloads from running on the master
+        ++ (pkgs.lib.optionals addTaints ["--node-taint=CriticalAddonsOnly=true:NoExecute"]);
+    in
+      pkgs.lib.concatStringsSep " " flagList;
   };
+
+  # create symlinks to link k3s's cni directory to the one used by almost all CNI plugins
+  systemd.tmpfiles.rules = [
+    "L+ /opt/cni/bin - - - - /var/lib/rancher/k3s/data/current/bin"
+    # seems like k3s's containerd will create /etc/cni/net.d, so we don't need to create a symlink for it
+    # "L+ /etc/cni/net.d - - - - /var/lib/rancher/k3s/agent/etc/cni/net.d"
+  ];
 }

modules/nixos/server/server.nix

@@ -3,4 +3,5 @@
     ../base
     ../../base.nix
   ];
+  boot.loader.timeout = 3; # wait for 3 seconds to select the boot entry
 }