feat: kubevirt on k3s

hosts/README.md

@@ -56,3 +56,5 @@ When building some packages for riscv64 or aarch64, I often have no cache availa
 
 ![](/_img/12kingdoms-1.webp)
 ![](/_img/12kingdoms-Youko-Rakushun.webp)
+
+[List of Frieren characters](https://en.wikipedia.org/wiki/List_of_Frieren_characters)

hosts/k8s/disko_config/README.md

@@ -0,0 +1,33 @@
+# Disko Config
+
+Generate LUKS keyfile to encrypt the root partition, it's used by disko.
+
+```bash
+# partition the usb stick
+parted /dev/sdb -- mklabel gpt
+parted /dev/sdb -- mkpart primary 2M 512MB
+parted /dev/sdb -- mkpart primary 512MB 1024MB
+mkfs.fat -F 32 -n NIXOS_DSC /dev/sdb1
+mkfs.fat -F 32 -n NIXOS_K3S /dev/sdb2
+
+# Generate a keyfile from the true random number generator
+KEYFILE=./kubevirt-luks-keyfile
+dd bs=8192 count=4 iflag=fullblock if=/dev/random of=$KEYFILE
+
+# generate token for k3s
+K3S_TOKEN_FILE=./kubevirt-k3s-token
+K3S_TOKEN=$(grep -ao '[A-Za-z0-9]' < /dev/random | head -64 | tr -d '\n' ; echo "")
+echo $K3S_TOKEN > $K3S_TOKEN_FILE
+
+# copy the keyfile and token to the usb stick
+
+KEYFILE=./kubevirt-luks-keyfile
+DEVICE=/dev/disk/by-label/NIXOS_DSC
+dd bs=8192 count=4 iflag=fullblock if=$KEYFILE of=$DEVICE
+
+K3S_TOKEN_FILE=./kubevirt-k3s-token
+USB_PATH=/run/media/ryan/NIXOS_K3S
+cp $K3S_TOKEN_FILE $USB_PATH
+```
+
+

hosts/k8s/disko_config/kubevirt-disko-fs.nix

@@ -0,0 +1,105 @@
+{
+  # contains the k3s's token
+  fileSystems."/run/media/nixos_k3s" = {
+    device = "/dev/disk/by-label/NIXOS_K3S";
+    fsType = "vfat";
+    mountOptions = [
+      "ro"
+    ];
+  };
+
+  disko.devices = {
+    disk = {
+      sda = {
+        type = "disk";
+        device = "/dev/nvme0n1";
+        content = {
+          type = "gpt";
+          partitions = {
+            # The EFI & Boot partition
+            ESP = {
+              size = "630M";
+              type = "EF00";
+              content = {
+                type = "filesystem";
+                format = "vfat";
+                mountpoint = "/boot/efi";
+                mountOptions = [
+                  "defaults"
+                ];
+              };
+            };
+            # The root partition
+            luks = {
+              size = "100%";
+              content = {
+                type = "luks";
+                name = "crypted";
+                settings = {
+                  keyFile = "/dev/disk/by-label/NIXOS_DSC"; # The keyfile is stored on a USB stick
+                  keyFileSize = 8192 * 4; # The maxium size of the keyfile is 8192 bytes
+                  keyFileOffset = 0;
+                  fallbackToPassword = true;
+                  allowDiscards = true;
+                };
+                # Whether to add a boot.initrd.luks.devices entry for the specified disk.
+                initrdUnlock = true;
+
+                # encrypt the root partition with luks2 and argon2id, will prompt for a passphrase, which will be used to unlock the partition.
+                # cryptsetup luksFormat
+                extraFormatArgs = [
+                  "--type luks2"
+                  "--cipher aes-xts-plain64"
+                  "--hash sha512"
+                  "--iter-time 5000"
+                  "--key-size 256"
+                  "--pbkdf argon2id"
+                  # use true random data from /dev/random, will block until enough entropy is available
+                  "--use-random"
+                ];
+                extraOpenArgs = [
+                  "--timeout 10"
+                ];
+                content = {
+                  type = "btrfs";
+                  extraArgs = ["-f"];
+                  subvolumes = {
+                    "@root" = {
+                      mountpoint = "/";
+                      mountOptions = ["compress-force=zstd:1" "noatime"];
+                    };
+                    "@home" = {
+                      mountpoint = "/home";
+                      mountOptions = ["compress-force=zstd:1"];
+                    };
+                    "@lib" = {
+                      mountpoint = "/var/lib";
+                      mountOptions = ["compress-force=zstd:1"];
+                    };
+
+                    "@nix" = {
+                      mountpoint = "/nix";
+                      mountOptions = ["compress-force=zstd:1" "noatime"];
+                    };
+                    "@tmp" = {
+                      mountpoint = "/tmp";
+                      mountOptions = ["compress-force=zstd:1" "noatime"];
+                    };
+                    "@snapshots" = {
+                      mountpoint = "/snapshots";
+                      mountOptions = ["compress-force=zstd:1" "noatime"];
+                    };
+                    "@swap" = {
+                      mountpoint = "/swap";
+                      swap.swapfile.size = "8192M";
+                    };
+                  };
+                };
+              };
+            };
+          };
+        };
+      };
+    };
+  };
+}

hosts/k8s/k3s_prod_1_master_1/default.nix

@@ -1,12 +1,13 @@
 {
+  pkgs,
   vars_networking,
   mylib,
   ...
 }: let
   hostName = "k3s-prod-1-master-1"; # Define your hostname.
   k8sLib = import ../lib.nix;
-  coreModule = k8sLib.genCoreModule {
-    inherit hostName vars_networking;
+  coreModule = k8sLib.gencoreModule {
+    inherit pkgs hostName vars_networking;
   };
 in {
   imports =

hosts/k8s/k3s_prod_1_master_1/k3s.nix

@@ -32,6 +32,8 @@ in {
       " --write-kubeconfig /home/${username}/.kube/config"
       + " --write-kubeconfig-mode 644"
       + " --service-node-port-range 80-32767"
+      + " --kube-apiserver-arg='--allow-privileged=true'" # required by kubevirt
+      + " --node-taint=CriticalAddonsOnly=true:NoExecute" # prevent workloads from running on the master
       + " --data-dir /var/lib/rancher/k3s"
       + " --disable-helm-controller"
       + " --etcd-expose-metrics true"

hosts/k8s/k3s_prod_1_master_2/default.nix

@@ -1,12 +1,13 @@
 {
+  pkgs,
   vars_networking,
   mylib,
   ...
 }: let
-  hostName = "k3s-prod-1-master-2"; # Define your hostname.
+  hostName = "k3s-prod-1-master-2"; # define your hostname.
   k8sLib = import ../lib.nix;
-  coreModule = k8sLib.genCoreModule {
-    inherit hostName vars_networking;
+  coreModule = k8sLib.gencoreModule {
+    inherit pkgs hostName vars_networking;
   };
 in {
   imports =

hosts/k8s/k3s_prod_1_master_2/k3s.nix

@@ -20,6 +20,8 @@ in {
       " --write-kubeconfig /etc/k3s/kubeconfig.yml"
       + " --write-kubeconfig-mode 644"
       + " --service-node-port-range 80-32767"
+      + " --kube-apiserver-arg='--allow-privileged=true'" # required by kubevirt
+      + " --node-taint=CriticalAddonsOnly=true:NoExecute" # prevent workloads from running on the master
       + " --data-dir /var/lib/rancher/k3s"
       + " --disable-helm-controller"
       + " --etcd-expose-metrics true"

hosts/k8s/k3s_prod_1_master_3/default.nix

@@ -1,12 +1,13 @@
 {
+  pkgs,
   vars_networking,
   mylib,
   ...
 }: let
-  hostName = "k3s-prod-1-master-3"; # Define your hostname.
+  hostName = "k3s-prod-1-master-3"; # define your hostname.
   k8sLib = import ../lib.nix;
-  coreModule = k8sLib.genCoreModule {
-    inherit hostName vars_networking;
+  coreModule = k8sLib.gencoreModule {
+    inherit pkgs hostName vars_networking;
   };
 in {
   imports =

hosts/k8s/k3s_prod_1_master_3/k3s.nix

@@ -20,6 +20,8 @@ in {
       " --write-kubeconfig /etc/k3s/kubeconfig.yml"
       + " --write-kubeconfig-mode 644"
       + " --service-node-port-range 80-32767"
+      + " --kube-apiserver-arg='--allow-privileged=true'" # required by kubevirt
+      + " --node-taint=CriticalAddonsOnly=true:NoExecute" # prevent workloads from running on the master
       + " --data-dir /var/lib/rancher/k3s"
       + " --disable-helm-controller"
       + " --etcd-expose-metrics true"

hosts/k8s/k3s_prod_1_worker_1/default.nix

@@ -1,12 +1,13 @@
 {
+  pkgs,
   vars_networking,
   mylib,
   ...
 }: let
-  hostName = "k3s-prod-1-worker-1"; # Define your hostname.
+  hostName = "k3s-prod-1-worker-1"; # define your hostname.
   k8sLib = import ../lib.nix;
-  coreModule = k8sLib.genCoreModule {
-    inherit hostName vars_networking;
+  coreModule = k8sLib.gencoreModule {
+    inherit pkgs hostName vars_networking;
   };
 in {
   imports =

hosts/k8s/k3s_prod_1_worker_1/k3s.nix

@@ -16,6 +16,8 @@ in {
     serverAddr = "https://${serverIp}:6443";
     tokenFile = config.age.secrets."k3s-prod-1-token".path;
     # https://docs.k3s.io/cli/agent
-    extraFlags = "--data-dir /var/lib/rancher/k3s";
+    extraFlags =
+      " --node-label=node-type=worker"
+      + " --data-dir /var/lib/rancher/k3s";
   };
 }

hosts/k8s/k3s_prod_1_worker_2/default.nix

@@ -1,12 +1,13 @@
 {
+  pkgs,
   vars_networking,
   mylib,
   ...
 }: let
-  hostName = "k3s-prod-1-worker-2"; # Define your hostname.
+  hostName = "k3s-prod-1-worker-2"; # define your hostname.
   k8sLib = import ../lib.nix;
-  coreModule = k8sLib.genCoreModule {
-    inherit hostName vars_networking;
+  coreModule = k8sLib.gencoreModule {
+    inherit pkgs hostName vars_networking;
   };
 in {
   imports =

hosts/k8s/k3s_prod_1_worker_2/k3s.nix

@@ -16,6 +16,8 @@ in {
     serverAddr = "https://${serverIp}:6443";
     tokenFile = config.age.secrets."k3s-prod-1-token".path;
     # https://docs.k3s.io/cli/agent
-    extraFlags = "--data-dir /var/lib/rancher/k3s";
+    extraFlags =
+      " --node-label=node-type=worker"
+      + " --data-dir /var/lib/rancher/k3s";
   };
 }

hosts/k8s/k3s_prod_1_worker_3/default.nix

@@ -1,12 +1,13 @@
 {
+  pkgs,
   vars_networking,
   mylib,
   ...
 }: let
-  hostName = "k3s-prod-1-worker-3"; # Define your hostname.
+  hostName = "k3s-prod-1-worker-3"; # define your hostname.
   k8sLib = import ../lib.nix;
-  coreModule = k8sLib.genCoreModule {
-    inherit hostName vars_networking;
+  coreModule = k8sLib.gencoreModule {
+    inherit pkgs hostName vars_networking;
   };
 in {
   imports =

hosts/k8s/k3s_prod_1_worker_3/k3s.nix

@@ -16,6 +16,8 @@ in {
     serverAddr = "https://${serverIp}:6443";
     tokenFile = config.age.secrets."k3s-prod-1-token".path;
     # https://docs.k3s.io/cli/agent
-    extraFlags = "--data-dir /var/lib/rancher/k3s";
+    extraFlags =
+      " --node-label=node-type=worker"
+      + " --data-dir /var/lib/rancher/k3s";
   };
 }

hosts/k8s/kubevirt_shoryu/default.nix

@@ -0,0 +1,22 @@
+{
+  pkgs,
+  mylib,
+  vars_networking,
+  disko,
+  ...
+}: let
+  # MoreFine - S500Plus
+  hostName = "kubevirt-shoryu"; # Define your hostname.
+  k8sLib = import ../lib.nix;
+  coreModule = k8sLib.gencoreModule {
+    inherit pkgs hostName vars_networking;
+  };
+in {
+  imports =
+    (mylib.scanPaths ./.)
+    ++ [
+      coreModule
+      disko.nixosModules.default
+      ../kubevirt-disko-fs.nix
+    ];
+}

hosts/k8s/kubevirt_shoryu/k3s.nix

@@ -0,0 +1,41 @@
+{
+  config,
+  pkgs,
+  username,
+  ...
+}: let
+  package = pkgs.k3s_1_29;
+in {
+  environment.systemPackages = with pkgs; [
+    package
+    k9s
+    kubectl
+    istioctl
+    kubernetes-helm
+
+    skopeo
+    dive # explore docker layers
+  ];
+  services.k3s = {
+    inherit package;
+    enable = true;
+
+    # Initialize HA cluster using an embedded etcd datastore.
+    # If you are configuring an HA cluster with an embedded etcd,
+    # the 1st server must have `clusterInit = true`
+    # and other servers must connect to it using serverAddr.
+    clusterInit = true;
+    role = "server";
+    tokenFile = "/run/media/nixos_k3s/kubevirt-k3s-token";
+    # https://docs.k3s.io/cli/server
+    extraFlags =
+      " --write-kubeconfig /etc/k3s/kubeconfig.yml"
+      + " --write-kubeconfig-mode 644"
+      + " --service-node-port-range 80-32767"
+      + " --kube-apiserver-arg='--allow-privileged=true'" # required by kubevirt
+      + " --data-dir /var/lib/rancher/k3s"
+      + " --disable-helm-controller"
+      + " --etcd-expose-metrics true"
+      + ''--etcd-snapshot-schedule-cron "0 */12 * * *"'';
+  };
+}

hosts/k8s/kubevirt_shushou/default.nix

@@ -0,0 +1,21 @@
+{
+  pkgs,
+  mylib,
+  vars_networking,
+  disko,
+  ...
+}: let
+  hostName = "kubevirt-shushou"; # Define your hostname.
+  k8sLib = import ../lib.nix;
+  coreModule = k8sLib.gencoreModule {
+    inherit pkgs hostName vars_networking;
+  };
+in {
+  imports =
+    (mylib.scanPaths ./.)
+    ++ [
+      coreModule
+      disko.nixosModules.default
+      ../kubevirt-disko-fs.nix
+    ];
+}

hosts/k8s/kubevirt_shushou/k3s.nix

@@ -0,0 +1,41 @@
+{
+  config,
+  pkgs,
+  username,
+  ...
+}: let
+  package = pkgs.k3s_1_29;
+in {
+  environment.systemPackages = with pkgs; [
+    package
+    k9s
+    kubectl
+    istioctl
+    kubernetes-helm
+
+    skopeo
+    dive # explore docker layers
+  ];
+  services.k3s = {
+    inherit package;
+    enable = true;
+
+    # Initialize HA cluster using an embedded etcd datastore.
+    # If you are configuring an HA cluster with an embedded etcd,
+    # the 1st server must have `clusterInit = true`
+    # and other servers must connect to it using serverAddr.
+    clusterInit = true;
+    role = "server";
+    tokenFile = "/run/media/nixos_k3s/kubevirt-k3s-token";
+    # https://docs.k3s.io/cli/server
+    extraFlags =
+      " --write-kubeconfig /etc/k3s/kubeconfig.yml"
+      + " --write-kubeconfig-mode 644"
+      + " --service-node-port-range 80-32767"
+      + " --kube-apiserver-arg='--allow-privileged=true'" # required by kubevirt
+      + " --data-dir /var/lib/rancher/k3s"
+      + " --disable-helm-controller"
+      + " --etcd-expose-metrics true"
+      + ''--etcd-snapshot-schedule-cron "0 */12 * * *"'';
+  };
+}

hosts/k8s/kubevirt_youko/default.nix

@@ -0,0 +1,21 @@
+{
+  pkgs,
+  mylib,
+  vars_networking,
+  disko,
+  ...
+}: let
+  hostName = "kubevirt-youko"; # Define your hostname.
+  k8sLib = import ../lib.nix;
+  coreModule = k8sLib.gencoreModule {
+    inherit pkgs hostName vars_networking;
+  };
+in {
+  imports =
+    (mylib.scanPaths ./.)
+    ++ [
+      coreModule
+      disko.nixosModules.default
+      ../kubevirt-disko-fs.nix
+    ];
+}

hosts/k8s/kubevirt_youko/k3s.nix

@@ -0,0 +1,41 @@
+{
+  config,
+  pkgs,
+  username,
+  ...
+}: let
+  package = pkgs.k3s_1_29;
+in {
+  environment.systemPackages = with pkgs; [
+    package
+    k9s
+    kubectl
+    istioctl
+    kubernetes-helm
+
+    skopeo
+    dive # explore docker layers
+  ];
+  services.k3s = {
+    inherit package;
+    enable = true;
+
+    # Initialize HA cluster using an embedded etcd datastore.
+    # If you are configuring an HA cluster with an embedded etcd,
+    # the 1st server must have `clusterInit = true`
+    # and other servers must connect to it using serverAddr.
+    clusterInit = true;
+    role = "server";
+    tokenFile = "/run/media/nixos_k3s/kubevirt-k3s-token";
+    # https://docs.k3s.io/cli/server
+    extraFlags =
+      " --write-kubeconfig /etc/k3s/kubeconfig.yml"
+      + " --write-kubeconfig-mode 644"
+      + " --service-node-port-range 80-32767"
+      + " --kube-apiserver-arg='--allow-privileged=true'" # required by kubevirt
+      + " --data-dir /var/lib/rancher/k3s"
+      + " --disable-helm-controller"
+      + " --etcd-expose-metrics true"
+      + ''--etcd-snapshot-schedule-cron "0 */12 * * *"'';
+  };
+}

hosts/k8s/lib.nix

@@ -1,7 +1,9 @@
 {
-  genCoreModule = {
+  gencoreModule = {
+    pkgs,
     hostName,
     vars_networking,
+    ...
   }: let
     hostAddress = vars_networking.hostAddress.${hostName};
   in {
@@ -18,9 +20,15 @@
       "cifs" # mount windows share
     ];
 
-    boot.kernelModules = ["kvm-amd"];
+    boot.kernelModules = ["kvm-amd" "vfio-pci"];
     boot.extraModprobeConfig = "options kvm_amd nested=1"; # for amd cpu
 
+    environment.systemPackages = with pkgs; [
+      # Validate Hardware Virtualization Support via:
+      #   virt-host-validate qemu
+      libvirt
+    ];
+
     networking = {
       inherit hostName;
       inherit (vars_networking) defaultGateway nameservers;