Merge pull request #102 from ryan4yin/k3s-test-cluter-running-on-kubevirt

feat: add a test k3s cluster running on kubevirt

Justfile

@@ -27,10 +27,11 @@ s-hypr mode="default":
   use utils.nu *; \
   nixos-switch shoukei-hyprland {{mode}}
 
-
+# Run eval tests
 test:
   nix eval .#evalTests --show-trace --print-build-logs --verbose
 
+# update all the flake inputs
 up:
   nix flake update
 
@@ -39,21 +40,25 @@ up:
 upp input:
   nix flake lock --update-input {{input}}
 
+# List all generations of the system profile
 history:
   nix profile history --profile /nix/var/nix/profiles/system
 
+# Open a nix shell with the flake
 repl:
   nix repl -f flake:nixpkgs
 
+# remove all generations older than 7 days
 clean:
-  # remove all generations older than 7 days
   sudo nix profile wipe-history --profile /nix/var/nix/profiles/system  --older-than 7d
 
+# Garbage collect all unused nix store entries
 gc:
   # garbage collect all unused nix store entries
   sudo nix store gc --debug
   sudo nix-collect-garbage --delete-old
 
+# Remove all reflog entries and prune unreachable objects
 gitgc:
   git reflog expire --expire-unreachable=now --all
   git gc --prune=now
@@ -72,46 +77,44 @@ darwin-rollback:
   use utils.nu *; \
   darwin-rollback
 
+# Deploy to harmonica(macOS host)
 ha mode="default":
   use utils.nu *; \
   darwin-build "harmonica" {{mode}}; \
   darwin-switch "harmonica" {{mode}}
 
+# Depoly to fern(macOS host)
 fe mode="default": darwin-set-proxy
   use utils.nu *; \
   darwin-build "fern" {{mode}}; \
   darwin-switch "fern" {{mode}}
 
+# Reload yabai and skhd(macOS)
 yabai-reload:
   launchctl kickstart -k "gui/502/org.nixos.yabai";
   launchctl kickstart -k "gui/502/org.nixos.skhd"; 
 
-############################################################################
-#
-#  Homelab - NixOS servers running on bare metal
-#
-############################################################################
-
-virt:
-  colmena apply --on '@virt-*' --verbose --show-trace
-
-shoryu:
-  colmena apply --on '@shoryu' --verbose --show-trace
-
-shushou:
-  colmena apply --on '@shushou' --verbose --show-trace
-
-youko:
-  colmena apply --on '@youko' --verbose --show-trace
-
-
 ############################################################################
 #
 #  Homelab - Virtual Machines running on Kubevirt
 #
 ############################################################################
 
+# Remote deployment via colmena
+col tag:
+  colmena apply --on '@{{tag}}' --verbose --show-trace
+
+# Build and upload a vm image
+upload-vm name mode="default":
+  use utils.nu *; \
+  upload-vm {{name}} {{mode}}
+
+# Deploy all the KubeVirt nodes(Physical machines running KubeVirt)
 lab:
+  colmena apply --on '@virt-*' --verbose --show-trace
+
+# Deploy all the VMs running on KubeVirt
+vm:
   colmena apply --on '@homelab-*' --verbose --show-trace
 
 aqua:
@@ -144,6 +147,8 @@ master:
 worker:
   colmena apply --on '@k3s-prod-1-worker-*' --verbose --show-trace
 
+k3s-test:
+  colmena apply --on '@k3s-test-*' --verbose --show-trace
 
 ############################################################################
 #
@@ -167,7 +172,7 @@ yukina:
 ############################################################################
 
 aarch:
-  colmena apply --on '@aarch' --verbose --show-trace
+  colmena apply --on '@aarch' --build-on-target --verbose --show-trace
 
 suzu:
   colmena apply --on '@suzu' --build-on-target --verbose --show-trace

hosts/12kingdoms-rakushun/caddy.nix

@@ -23,6 +23,23 @@
       encode zstd gzip
       reverse_proxy http://localhost:3000
     '';
+
+    # https://caddyserver.com/docs/caddyfile/directives/file_server
+    virtualHosts."http://file.writefor.fun".extraConfig = ''
+      root * /var/lib/caddy/fileserver/
+      encode zstd gzip
+      file_server browse {
+        hide .git
+        precompressed zstd br gzip
+      }
+    '';
   };
   networking.firewall.allowedTCPPorts = [80 443];
+
+  # Create Directories
+  systemd.tmpfiles.rules = [
+    "d /var/lib/caddy/fileserver/ 0755 caddy caddy"
+    # directory for virual machine's images
+    "d /var/lib/caddy/fileserver/vms 0755 caddy caddy"
+  ];
 }

hosts/README.md

@@ -38,6 +38,9 @@
    4. [Optional] Add a new integration test file under
       `outputs/<system-architecture>/integration-tests/<name>.nix` to test whether the new host's
       nix config can be built and deployed correctly.
+1. Under `vars/networking.nix`
+   1. Add the new host's static IP address.
+   1. Skip this step if the new host is not in the local network or is a mobile device.
 
 ## idols - Oshi no Ko
 

hosts/k8s/k3s-test-1-master-1/default.nix

@@ -0,0 +1,28 @@
+{
+  config,
+  pkgs,
+  myvars,
+  mylib,
+  ...
+}: let
+  hostName = "k3s-test-1-master-1"; # Define your hostname.
+
+  coreModule = mylib.genKubeVirtCoreModule {
+    inherit pkgs hostName;
+    inherit (myvars) networking;
+  };
+  k3sModule = mylib.genK3sServerModule {
+    inherit pkgs;
+    kubeconfigFile = "/home/${myvars.username}/.kube/config";
+    tokenFile = config.age.secrets."k3s-prod-1-token".path;
+    # the first node in the cluster should be the one to initialize the cluster
+    clusterInit = true;
+  };
+in {
+  imports =
+    (mylib.scanPaths ./.)
+    ++ [
+      coreModule
+      k3sModule
+    ];
+}

hosts/k8s/k3s-test-1-master-2/default.nix

@@ -0,0 +1,28 @@
+{
+  config,
+  pkgs,
+  myvars,
+  mylib,
+  ...
+}: let
+  hostName = "k3s-test-1-master-2"; # define your hostname.
+  k3sServerName = "k3s-test-1-master-1";
+
+  coreModule = mylib.genKubeVirtCoreModule {
+    inherit pkgs hostName;
+    inherit (myvars) networking;
+  };
+  k3sModule = mylib.genK3sServerModule {
+    inherit pkgs;
+    kubeconfigFile = "/home/${myvars.username}/.kube/config";
+    tokenFile = config.age.secrets."k3s-prod-1-token".path;
+    serverIp = myvars.networking.hostsAddr.${k3sServerName}.ipv4;
+  };
+in {
+  imports =
+    (mylib.scanPaths ./.)
+    ++ [
+      coreModule
+      k3sModule
+    ];
+}

hosts/k8s/k3s-test-1-master-3/default.nix

@@ -0,0 +1,28 @@
+{
+  config,
+  pkgs,
+  myvars,
+  mylib,
+  ...
+}: let
+  hostName = "k3s-test-1-master-3"; # define your hostname.
+  k3sServerName = "k3s-test-1-master-1";
+
+  coreModule = mylib.genKubeVirtCoreModule {
+    inherit pkgs hostName;
+    inherit (myvars) networking;
+  };
+  k3sModule = mylib.genK3sServerModule {
+    inherit pkgs;
+    kubeconfigFile = "/home/${myvars.username}/.kube/config";
+    tokenFile = config.age.secrets."k3s-prod-1-token".path;
+    serverIp = myvars.networking.hostsAddr.${k3sServerName}.ipv4;
+  };
+in {
+  imports =
+    (mylib.scanPaths ./.)
+    ++ [
+      coreModule
+      k3sModule
+    ];
+}

modules/nixos/server/kubevirt-hardware-configuration.nix

@@ -0,0 +1,34 @@
+{modulesPath, ...}:
+##############################################################################
+#
+#  Template for KubeVirt's VM, mainly based on:
+#    https://github.com/NixOS/nixpkgs/blob/master/nixos/modules/virtualisation/kubevirt.nix
+#
+#  We write our hardware-configuration.nix, so that we can do some customization more easily.
+#
+#  the url above is used by `nixos-generator` to generate the KubeVirt's qcow2 image file.
+#
+##############################################################################
+{
+  imports = [
+    "${toString modulesPath}/profiles/qemu-guest.nix"
+  ];
+
+  config = {
+    fileSystems."/" = {
+      device = "/dev/disk/by-label/nixos";
+      fsType = "ext4";
+      autoResize = true;
+    };
+
+    boot.growPartition = true;
+    boot.kernelParams = [ "console=ttyS0" ];
+    boot.loader.grub.device = "/dev/vda";
+    boot.loader.timeout = 0;
+
+    services.qemuGuest.enable = true;
+    services.openssh.enable = true;
+    services.cloud-init.enable = true;
+    systemd.services."serial-getty@ttyS0".enable = true;
+  };
+}

outputs/x86_64-linux/src/k3s-test-1-master-1.nix

@@ -0,0 +1,44 @@
+{
+  # NOTE: the args not used in this file CAN NOT be removed!
+  # because haumea pass argument lazily,
+  # and these arguments are used in the functions like `mylib.nixosSystem`, `mylib.colmenaSystem`, etc.
+  inputs,
+  lib,
+  mylib,
+  myvars,
+  system,
+  genSpecialArgs,
+  ...
+} @ args: let
+  name = "k3s-test-1-master-1";
+  tags = [name];
+  ssh-user = "root";
+
+  modules = {
+    nixos-modules =
+      (map mylib.relativeToRoot [
+        # common
+        "secrets/nixos.nix"
+        "modules/nixos/server/server.nix"
+        "modules/nixos/server/kubevirt-hardware-configuration.nix"
+        # host specific
+        "hosts/k8s/${name}"
+      ])
+      ++ [
+        {modules.secrets.server.kubernetes.enable = true;}
+      ];
+    home-modules = map mylib.relativeToRoot [
+      "home/linux/core.nix"
+    ];
+  };
+
+  systemArgs = modules // args;
+in {
+  nixosConfigurations.${name} = mylib.nixosSystem systemArgs;
+
+  colmena.${name} =
+    mylib.colmenaSystem (systemArgs // {inherit tags ssh-user;});
+
+  # generate proxmox image for virtual machines without desktop environment
+  packages.${name} = inputs.self.nixosConfigurations.${name}.config.formats.kubevirt;
+}

outputs/x86_64-linux/src/k3s-test-1-master-2.nix

@@ -0,0 +1,41 @@
+{
+  # NOTE: the args not used in this file CAN NOT be removed!
+  # because haumea pass argument lazily,
+  # and these arguments are used in the functions like `mylib.nixosSystem`, `mylib.colmenaSystem`, etc.
+  inputs,
+  lib,
+  mylib,
+  myvars,
+  system,
+  genSpecialArgs,
+  ...
+} @ args: let
+  name = "k3s-test-1-master-2";
+  tags = [name];
+  ssh-user = "root";
+
+  modules = {
+    nixos-modules =
+      (map mylib.relativeToRoot [
+        # common
+        "secrets/nixos.nix"
+        "modules/nixos/server/server.nix"
+        "modules/nixos/server/kubevirt-hardware-configuration.nix"
+        # host specific
+        "hosts/k8s/${name}"
+      ])
+      ++ [
+        {modules.secrets.server.kubernetes.enable = true;}
+      ];
+  };
+
+  systemArgs = modules // args;
+in {
+  nixosConfigurations.${name} = mylib.nixosSystem systemArgs;
+
+  colmena.${name} =
+    mylib.colmenaSystem (systemArgs // {inherit tags ssh-user;});
+
+  # generate proxmox image for virtual machines without desktop environment
+  packages.${name} = inputs.self.nixosConfigurations.${name}.config.formats.kubevirt;
+}

outputs/x86_64-linux/src/k3s-test-1-master-3.nix

@@ -0,0 +1,41 @@
+{
+  # NOTE: the args not used in this file CAN NOT be removed!
+  # because haumea pass argument lazily,
+  # and these arguments are used in the functions like `mylib.nixosSystem`, `mylib.colmenaSystem`, etc.
+  inputs,
+  lib,
+  mylib,
+  myvars,
+  system,
+  genSpecialArgs,
+  ...
+} @ args: let
+  name = "k3s-test-1-master-3";
+  tags = [name];
+  ssh-user = "root";
+
+  modules = {
+    nixos-modules =
+      (map mylib.relativeToRoot [
+        # common
+        "secrets/nixos.nix"
+        "modules/nixos/server/server.nix"
+        "modules/nixos/server/kubevirt-hardware-configuration.nix"
+        # host specific
+        "hosts/k8s/${name}"
+      ])
+      ++ [
+        {modules.secrets.server.kubernetes.enable = true;}
+      ];
+  };
+
+  systemArgs = modules // args;
+in {
+  nixosConfigurations.${name} = mylib.nixosSystem systemArgs;
+
+  colmena.${name} =
+    mylib.colmenaSystem (systemArgs // {inherit tags ssh-user;});
+
+  # generate proxmox image for virtual machines without desktop environment
+  packages.${name} = inputs.self.nixosConfigurations.${name}.config.formats.kubevirt;
+}

secrets/nixos.nix

@@ -236,6 +236,12 @@ in {
               file = "${mysecrets}/server/k3s-prod-1-token.age";
             }
             // high_security;
+
+          "k3s-test-1-token" =
+            {
+              file = "${mysecrets}/server/k3s-test-1-token.age";
+            }
+            // high_security;
         };
       })
     ]);

utils.nu

@@ -53,3 +53,22 @@ export def darwin-switch [
 export def darwin-rollback [] {
     ./result/sw/bin/darwin-rebuild --rollback
 }
+
+# ==================== Virutal Machines related =====================
+
+# Build and upload a VM image
+export def upload-vm [
+    name: string
+    mode: string
+] {
+    let target = $".#($name)"
+    if "debug" == $mode {
+        nom build $target --show-trace --verbose
+    } else {
+        nix build $target
+    }
+
+    let remote = $"root@rakushun:/var/lib/caddy/fileserver/vms/kubevirt-($name).qcow2"
+    rsync -avz --progress --copy-links result $remote
+}
+

vars/networking.nix

@@ -108,6 +108,22 @@
       iface = "ens18";
       ipv4 = "192.168.5.113";
     };
+
+    k3s-test-1-master-1 = {
+      # KubeVirt VM
+      iface = "eth1";
+      ipv4 = "192.168.5.114";
+    };
+    k3s-test-1-master-2 = {
+      # KubeVirt VM
+      iface = "eth1";
+      ipv4 = "192.168.5.115";
+    };
+    k3s-test-1-master-3 = {
+      # KubeVirt VM
+      iface = "eth1";
+      ipv4 = "192.168.5.116";
+    };
   };
 
   hostsInterface =