From cf2435e8545232f52e9dd9959ea2987631df20ea Mon Sep 17 00:00:00 2001 From: Ryan Yin Date: Mon, 21 Sep 2026 16:57:42 +0800 Subject: [PATCH] chore(hosts): retire the k3s cluster on the physical hosts The physical hosts no longer need k3s or KubeVirt: aquamarine runs natively, the k3s-test masters run as microVMs and kana/ruby under libvirt. So drop the host cluster and split the shared parts out of the KubeVirt host module: - lib/genVmHostModule.nix: the host bits the VMs still need (br0 + the physical NIC as a bridge port, KVM, sysctls, earlyoom, libvirt, journald, stateVersion) - the three hosts use it and drop genK3sServerModule; also drop the longhorn workarounds (openiscsi + the /usr/local/bin tmpfile) and the virtctl package - remove lib/genKubeVirtHostModule.nix --- hosts/k8s/kubevirt-shoryu/default.nix | 32 +----------------- hosts/k8s/kubevirt-shushou/default.nix | 30 +---------------- hosts/k8s/kubevirt-youko/default.nix | 30 +---------------- lib/default.nix | 2 +- ...VirtHostModule.nix => genVmHostModule.nix} | 33 ++++--------------- 5 files changed, 11 insertions(+), 116 deletions(-) rename lib/{genKubeVirtHostModule.nix => genVmHostModule.nix} (78%) diff --git a/hosts/k8s/kubevirt-shoryu/default.nix b/hosts/k8s/kubevirt-shoryu/default.nix index 7043f500..360b987d 100644 --- a/hosts/k8s/kubevirt-shoryu/default.nix +++ b/hosts/k8s/kubevirt-shoryu/default.nix @@ -10,39 +10,10 @@ let # MoreFine - S500Plus hostName = "kubevirt-shoryu"; # Define your hostname. - coreModule = mylib.genKubeVirtHostModule { + coreModule = mylib.genVmHostModule { inherit pkgs hostName; inherit (myvars) networking; }; - k3sModule = mylib.genK3sServerModule { - inherit pkgs; - kubeconfigFile = "/home/${myvars.username}/.kube/config"; - tokenFile = "/persistent/kubevirt-k3s-token"; - # the first node in the cluster should be the one to initialize the cluster - clusterInit = true; - # use my own domain & kube-vip's virtual IP for the API server - # so that the API server can always be accessed even if some nodes are down - masterHost = "kubevirt-cluster-1.writefor.fun"; - kubeletExtraArgs = [ - "--cpu-manager-policy=static" - # https://kubernetes.io/docs/tasks/administer-cluster/reserve-compute-resources/ - # we have to reserve some resources for for system daemons running as pods or system services - # when cpu-manager's static policy is enabled - # the memory we reserved here is also for the kernel, since kernel's memory is not accounted in pods - "--system-reserved=cpu=1,memory=2Gi,ephemeral-storage=2Gi" - ]; - k3sExtraArgs = [ - # IPv4 Private CIDR(full) - 172.16.0.0/12 - # IPv4 Pod CIDR(full) - fdfd:cafe:00:0000::/64 ~ fdfd:cafe:00:7fff::/64 - # IPv4 Service CIDR(full) - fdfd:cafe:00:8000::/64 ~ fdfd:cafe:00:ffff::/64 - # "--cluster-cidr=172.16.0.0/16,fdfd:cafe:00:0001::/64" - # "--service-cidr=172.17.0.0/16,fdfd:cafe:00:8001::/112" - ]; - nodeLabels = [ - "node-purpose=kubevirt" - ]; - # flannel is disabled; Cilium provides the pod network. - }; in { imports = (mylib.scanPaths ./.) ++ [ @@ -51,7 +22,6 @@ in ./hardware-configuration.nix ./preservation.nix coreModule - k3sModule ]; modules.btrbk.enable = true; diff --git a/hosts/k8s/kubevirt-shushou/default.nix b/hosts/k8s/kubevirt-shushou/default.nix index e6754772..90bb7a17 100644 --- a/hosts/k8s/kubevirt-shushou/default.nix +++ b/hosts/k8s/kubevirt-shushou/default.nix @@ -9,37 +9,10 @@ let hostName = "kubevirt-shushou"; # Define your hostname. - coreModule = mylib.genKubeVirtHostModule { + coreModule = mylib.genVmHostModule { inherit pkgs hostName; inherit (myvars) networking; }; - k3sModule = mylib.genK3sServerModule { - inherit pkgs; - kubeconfigFile = "/home/${myvars.username}/.kube/config"; - tokenFile = "/persistent/kubevirt-k3s-token"; - # use my own domain & kube-vip's virtual IP for the API server - # so that the API server can always be accessed even if some nodes are down - masterHost = "kubevirt-cluster-1.writefor.fun"; - kubeletExtraArgs = [ - "--cpu-manager-policy=static" - # https://kubernetes.io/docs/tasks/administer-cluster/reserve-compute-resources/ - # we have to reserve some resources for for system daemons running as pods or system services - # when cpu-manager's static policy is enabled - # the memory we reserved here is also for the kernel, since kernel's memory is not accounted in pods - "--system-reserved=cpu=1,memory=2Gi,ephemeral-storage=2Gi" - ]; - k3sExtraArgs = [ - # IPv4 Private CIDR(full) - 172.16.0.0/12 - # IPv4 Pod CIDR(full) - fdfd:cafe:00:0000::/64 ~ fdfd:cafe:00:7fff::/64 - # IPv4 Service CIDR(full) - fdfd:cafe:00:8000::/64 ~ fdfd:cafe:00:ffff::/64 - # "--cluster-cidr=172.16.0.0/16,fdfd:cafe:00:0001::/64" - # "--service-cidr=172.17.0.0/16,fdfd:cafe:00:8001::/112" - ]; - nodeLabels = [ - "node-purpose=kubevirt" - ]; - # flannel is disabled; Cilium provides the pod network. - }; in { imports = (mylib.scanPaths ./.) ++ [ @@ -48,7 +21,6 @@ in ../kubevirt-shoryu/hardware-configuration.nix ../kubevirt-shoryu/preservation.nix coreModule - k3sModule ]; modules.btrbk.enable = true; diff --git a/hosts/k8s/kubevirt-youko/default.nix b/hosts/k8s/kubevirt-youko/default.nix index c77bd7fe..54d3d127 100644 --- a/hosts/k8s/kubevirt-youko/default.nix +++ b/hosts/k8s/kubevirt-youko/default.nix @@ -9,37 +9,10 @@ let hostName = "kubevirt-youko"; # Define your hostname. - coreModule = mylib.genKubeVirtHostModule { + coreModule = mylib.genVmHostModule { inherit pkgs hostName; inherit (myvars) networking; }; - k3sModule = mylib.genK3sServerModule { - inherit pkgs; - kubeconfigFile = "/home/${myvars.username}/.kube/config"; - tokenFile = "/persistent/kubevirt-k3s-token"; - # use my own domain & kube-vip's virtual IP for the API server - # so that the API server can always be accessed even if some nodes are down - masterHost = "kubevirt-cluster-1.writefor.fun"; - kubeletExtraArgs = [ - "--cpu-manager-policy=static" - # https://kubernetes.io/docs/tasks/administer-cluster/reserve-compute-resources/ - # we have to reserve some resources for for system daemons running as pods or system services - # when cpu-manager's static policy is enabled - # the memory we reserved here is also for the kernel, since kernel's memory is not accounted in pods - "--system-reserved=cpu=1,memory=2Gi,ephemeral-storage=2Gi" - ]; - k3sExtraArgs = [ - # IPv4 Private CIDR(full) - 172.16.0.0/12 - # IPv4 Pod CIDR(full) - fdfd:cafe:00:0000::/64 ~ fdfd:cafe:00:7fff::/64 - # IPv4 Service CIDR(full) - fdfd:cafe:00:8000::/64 ~ fdfd:cafe:00:ffff::/64 - # "--cluster-cidr=172.16.0.0/16,fdfd:cafe:00:0001::/64" - # "--service-cidr=172.17.0.0/16,fdfd:cafe:00:8001::/112" - ]; - nodeLabels = [ - "node-purpose=kubevirt" - ]; - # flannel is disabled; Cilium provides the pod network. - }; in { imports = (mylib.scanPaths ./.) ++ [ @@ -48,7 +21,6 @@ in ../kubevirt-shoryu/hardware-configuration.nix ../kubevirt-shoryu/preservation.nix coreModule - k3sModule ]; modules.btrbk.enable = true; diff --git a/lib/default.nix b/lib/default.nix index 3f762523..f72ffb6d 100644 --- a/lib/default.nix +++ b/lib/default.nix @@ -8,7 +8,7 @@ genK3sServerModule = import ./genK3sServerModule.nix; genK3sAgentModule = import ./genK3sAgentModule.nix; - genKubeVirtHostModule = import ./genKubeVirtHostModule.nix; + genVmHostModule = import ./genVmHostModule.nix; genKubeVirtGuestModule = import ./genKubeVirtGuestModule.nix; genMicrovmGuestModule = import ./genMicrovmGuestModule.nix; diff --git a/lib/genKubeVirtHostModule.nix b/lib/genVmHostModule.nix similarity index 78% rename from lib/genKubeVirtHostModule.nix rename to lib/genVmHostModule.nix index 5c41a79e..2b2c0b28 100644 --- a/lib/genKubeVirtHostModule.nix +++ b/lib/genVmHostModule.nix @@ -5,7 +5,7 @@ ... }: let - inherit (networking) proxyGateway proxyGateway6 nameservers; + inherit (networking) proxyGateway proxyGateway6; inherit (networking.hostsAddr.${hostName}) iface ipv4; ipv4WithMask = "${ipv4}/24"; in @@ -20,13 +20,11 @@ in "fat" "vfat" "exfat" - "nfs" # required by longhorn + "nfs" ]; - boot.kernelModules = [ - "kvm-amd" - "vfio-pci" - ]; + # KVM for the microVMs and the libvirt domains. + boot.kernelModules = [ "kvm-amd" ]; boot.extraModprobeConfig = "options kvm_amd nested=1"; # for amd cpu boot.kernel.sysctl = { @@ -52,33 +50,18 @@ in # NOTE: vm.swappiness is intentionally NOT set here; it comes from # modules/nixos/base/zram.nix (mkDefault 180, tuned for the zram device). - # The previous hard-coded 0 disabled swapping entirely, so the zram device - # (already enabled by that module) was never used. }; # zram itself is provided by modules/nixos/base/zram.nix (enabled by default). # Kill the greediest process before the host starts thrashing / gets OOM-killed. - # (defaults: trigger at 10% free mem / swap, kill hard at half of that.) services.earlyoom.enable = true; environment.systemPackages = with pkgs; [ # Validate Hardware Virtualization Support via: # virt-host-validate qemu libvirt - kubevirt # virtctl ]; - # Workaround for longhorn running on NixOS - # https://github.com/longhorn/longhorn/issues/2166 - systemd.tmpfiles.rules = [ - "L+ /usr/local/bin - - - - /run/current-system/sw/bin/" - ]; - # Longhorn uses open-iscsi to create block devices. - services.openiscsi = { - name = "iqn.2020-08.org.linux-iscsi.initiatorhost:${hostName}"; - enable = true; - }; - networking = { inherit hostName; @@ -89,9 +72,9 @@ in networking.useNetworkd = true; systemd.network.enable = true; - # Linux bridge for the VMs' secondary network. It replaces the previous OVS - # bridge: the `bridge` CNI plugin attaches the VM taps to br0, and the host - # keeps its own address on br0 (the physical NIC is just a bridge port). + # Linux bridge for the VMs: the microVM taps and the libvirt domains attach to + # br0, and the host keeps its own address on it (the physical NIC is just a + # bridge port). systemd.network.netdevs."10-br0" = { netdevConfig = { Name = "br0"; @@ -99,8 +82,6 @@ in }; }; - # systemd.services."systemd-networkd".environment.SYSTEMD_LOG_LEVEL = "debug"; - # Set the host's address on the bridge interface instead of the physical interface! systemd.network.networks = { "10-br0" = {