From 4ad359696e3892fc30a36aabd7a5ab73be0fd38d Mon Sep 17 00:00:00 2001 From: Ryan Yin Date: Mon, 21 Sep 2026 16:32:23 +0800 Subject: [PATCH] feat(microvm): run the k3s-test-1 masters as microVMs Replace the KubeVirt/etcd members with microVMs on the three physical hosts, keeping the existing cluster: the guests inherit their host key and k3s state from the old disks, so etcd recovers in place instead of bootstrapping a new cluster. - convert hosts/k8s/k3s-test-1-master-{1,2,3} to genMicrovmGuestModule and switch their outputs to the microvm module (keep home-modules) - declare the vms on shoryu/youko/shushou and bridge their taps to br0 - each guest config is self-contained (its secret group and the nixpkgs.config override live in the guest, not the host module) - guest specialArgs carry agenix/mysecrets so secrets/nixos.nix works --- hosts/k8s/k3s-test-1-master-1/default.nix | 27 +++++++++++--- hosts/k8s/k3s-test-1-master-2/default.nix | 37 ++++++++++++------- hosts/k8s/k3s-test-1-master-3/default.nix | 37 ++++++++++++------- hosts/k8s/kubevirt-shoryu/microvm.nix | 37 +++++++++++++++++++ hosts/k8s/kubevirt-shushou/microvm.nix | 37 +++++++++++++++++++ hosts/k8s/kubevirt-youko/microvm.nix | 37 +++++++++++++++++++ .../x86_64-linux/src/k3s-test-1-master-1.nix | 10 ++--- .../x86_64-linux/src/k3s-test-1-master-2.nix | 13 +++---- .../x86_64-linux/src/k3s-test-1-master-3.nix | 13 +++---- 9 files changed, 194 insertions(+), 54 deletions(-) create mode 100644 hosts/k8s/kubevirt-shoryu/microvm.nix create mode 100644 hosts/k8s/kubevirt-shushou/microvm.nix create mode 100644 hosts/k8s/kubevirt-youko/microvm.nix diff --git a/hosts/k8s/k3s-test-1-master-1/default.nix b/hosts/k8s/k3s-test-1-master-1/default.nix index f9a5f87b..d2ac8c29 100644 --- a/hosts/k8s/k3s-test-1-master-1/default.nix +++ b/hosts/k8s/k3s-test-1-master-1/default.nix @@ -1,6 +1,7 @@ { config, pkgs, + lib, myvars, mylib, ... @@ -8,9 +9,11 @@ let hostName = "k3s-test-1-master-1"; # Define your hostname. - coreModule = mylib.genKubeVirtGuestModule { + coreModule = mylib.genMicrovmGuestModule { inherit pkgs hostName; inherit (myvars) networking; + vcpu = 2; + mem = 8192; }; k3sModule = mylib.genK3sServerModule { inherit pkgs; @@ -35,8 +38,22 @@ let }; in { - imports = (mylib.scanPaths ./.) ++ [ - coreModule - k3sModule - ]; + imports = + (mylib.scanPaths ./.) + ++ (map mylib.relativeToRoot [ + "secrets/nixos.nix" + "modules/nixos/server/server.nix" + ]) + ++ [ + coreModule + k3sModule + ]; + + # k3s-test-1-token lives in this secret group + modules.secrets.server.kubernetes.enable = true; + + # The guest reuses the host's already-instantiated nixpkgs (shared store), + # so nixpkgs.config may not be set. The shared base modules set + # nixpkgs.config.allowUnfree, so force it empty here. + nixpkgs.config = lib.mkForce { }; } diff --git a/hosts/k8s/k3s-test-1-master-2/default.nix b/hosts/k8s/k3s-test-1-master-2/default.nix index 8f60aa7d..14397dec 100644 --- a/hosts/k8s/k3s-test-1-master-2/default.nix +++ b/hosts/k8s/k3s-test-1-master-2/default.nix @@ -1,16 +1,19 @@ { config, pkgs, + lib, myvars, mylib, ... }: let - hostName = "k3s-test-1-master-2"; # define your hostname. + hostName = "k3s-test-1-master-2"; # Define your hostname. - coreModule = mylib.genKubeVirtGuestModule { + coreModule = mylib.genMicrovmGuestModule { inherit pkgs hostName; inherit (myvars) networking; + vcpu = 2; + mem = 4096; }; k3sModule = mylib.genK3sServerModule { inherit pkgs; @@ -22,19 +25,25 @@ let # the control plane only runs the control plane; workloads run on the # k3s-test-1-worker-* nodes (labelled node-role.kubernetes.io/worker=true) nodeTaints = [ "node-role.kubernetes.io/control-plane:NoSchedule" ]; - - # k3sExtraArgs = [ - # # IPv4 Private CIDR(full) - 172.16.0.0/12 - # # IPv4 Pod CIDR(full) - fdfd:cafe:00:0000::/64 ~ fdfd:cafe:00:7fff::/64 - # # IPv4 Service CIDR(full) - fdfd:cafe:00:8000::/64 ~ fdfd:cafe:00:ffff::/64 - # "--cluster-cidr=172.18.0.0/16,fdfd:cafe:00:0002::/64" - # "--service-cidr=172.19.0.0/16,fdfd:cafe:00:8002::/112" - # ]; }; in { - imports = (mylib.scanPaths ./.) ++ [ - coreModule - k3sModule - ]; + imports = + (mylib.scanPaths ./.) + ++ (map mylib.relativeToRoot [ + "secrets/nixos.nix" + "modules/nixos/server/server.nix" + ]) + ++ [ + coreModule + k3sModule + ]; + + # k3s-test-1-token lives in this secret group + modules.secrets.server.kubernetes.enable = true; + + # The guest reuses the host's already-instantiated nixpkgs (shared store), + # so nixpkgs.config may not be set. The shared base modules set + # nixpkgs.config.allowUnfree, so force it empty here. + nixpkgs.config = lib.mkForce { }; } diff --git a/hosts/k8s/k3s-test-1-master-3/default.nix b/hosts/k8s/k3s-test-1-master-3/default.nix index d11e1e7b..6f5c0eda 100644 --- a/hosts/k8s/k3s-test-1-master-3/default.nix +++ b/hosts/k8s/k3s-test-1-master-3/default.nix @@ -1,16 +1,19 @@ { config, pkgs, + lib, myvars, mylib, ... }: let - hostName = "k3s-test-1-master-3"; # define your hostname. + hostName = "k3s-test-1-master-3"; # Define your hostname. - coreModule = mylib.genKubeVirtGuestModule { + coreModule = mylib.genMicrovmGuestModule { inherit pkgs hostName; inherit (myvars) networking; + vcpu = 2; + mem = 8192; }; k3sModule = mylib.genK3sServerModule { inherit pkgs; @@ -22,19 +25,25 @@ let # the control plane only runs the control plane; workloads run on the # k3s-test-1-worker-* nodes (labelled node-role.kubernetes.io/worker=true) nodeTaints = [ "node-role.kubernetes.io/control-plane:NoSchedule" ]; - - # k3sExtraArgs = [ - # # IPv4 Private CIDR(full) - 172.16.0.0/12 - # # IPv4 Pod CIDR(full) - fdfd:cafe:00:0000::/64 ~ fdfd:cafe:00:7fff::/64 - # # IPv4 Service CIDR(full) - fdfd:cafe:00:8000::/64 ~ fdfd:cafe:00:ffff::/64 - # "--cluster-cidr=172.18.0.0/16,fdfd:cafe:00:0002::/64" - # "--service-cidr=172.19.0.0/16,fdfd:cafe:00:8002::/112" - # ]; }; in { - imports = (mylib.scanPaths ./.) ++ [ - coreModule - k3sModule - ]; + imports = + (mylib.scanPaths ./.) + ++ (map mylib.relativeToRoot [ + "secrets/nixos.nix" + "modules/nixos/server/server.nix" + ]) + ++ [ + coreModule + k3sModule + ]; + + # k3s-test-1-token lives in this secret group + modules.secrets.server.kubernetes.enable = true; + + # The guest reuses the host's already-instantiated nixpkgs (shared store), + # so nixpkgs.config may not be set. The shared base modules set + # nixpkgs.config.allowUnfree, so force it empty here. + nixpkgs.config = lib.mkForce { }; } diff --git a/hosts/k8s/kubevirt-shoryu/microvm.nix b/hosts/k8s/kubevirt-shoryu/microvm.nix new file mode 100644 index 00000000..fca62fb8 --- /dev/null +++ b/hosts/k8s/kubevirt-shoryu/microvm.nix @@ -0,0 +1,37 @@ +{ + microvm, + myvars, + mylib, + agenix, + mysecrets, + ... +}: +{ + # Run the k3s-test VMs as MicroVMs instead of KubeVirt domains. + imports = [ microvm.nixosModules.host ]; + + microvm.vms.k3s-test-1-master-1 = { + autostart = true; + restartIfChanged = true; + specialArgs = { + inherit + myvars + mylib + agenix + mysecrets + ; + }; + config.imports = [ ../k3s-test-1-master-1 ]; + }; + + # Attach the guest's tap to the VM bridge, the same way the physical NIC is + # attached. The tap name is derived from the guest IP (192.168.5.114 -> vm114). + systemd.network.networks."20-vm114" = { + matchConfig.Name = [ "vm114" ]; + networkConfig = { + LinkLocalAddressing = "no"; + Bridge = "br0"; + }; + linkConfig.RequiredForOnline = "no"; + }; +} diff --git a/hosts/k8s/kubevirt-shushou/microvm.nix b/hosts/k8s/kubevirt-shushou/microvm.nix new file mode 100644 index 00000000..72871125 --- /dev/null +++ b/hosts/k8s/kubevirt-shushou/microvm.nix @@ -0,0 +1,37 @@ +{ + microvm, + myvars, + mylib, + agenix, + mysecrets, + ... +}: +{ + # Run the k3s-test VMs as MicroVMs instead of KubeVirt domains. + imports = [ microvm.nixosModules.host ]; + + microvm.vms.k3s-test-1-master-3 = { + autostart = true; + restartIfChanged = true; + specialArgs = { + inherit + myvars + mylib + agenix + mysecrets + ; + }; + config.imports = [ ../k3s-test-1-master-3 ]; + }; + + # Attach the guest's tap to the VM bridge, the same way the physical NIC is + # attached. The tap name is derived from the guest IP (192.168.5.116 -> vm116). + systemd.network.networks."20-vm116" = { + matchConfig.Name = [ "vm116" ]; + networkConfig = { + LinkLocalAddressing = "no"; + Bridge = "br0"; + }; + linkConfig.RequiredForOnline = "no"; + }; +} diff --git a/hosts/k8s/kubevirt-youko/microvm.nix b/hosts/k8s/kubevirt-youko/microvm.nix new file mode 100644 index 00000000..96150b0d --- /dev/null +++ b/hosts/k8s/kubevirt-youko/microvm.nix @@ -0,0 +1,37 @@ +{ + microvm, + myvars, + mylib, + agenix, + mysecrets, + ... +}: +{ + # Run the k3s-test VMs as MicroVMs instead of KubeVirt domains. + imports = [ microvm.nixosModules.host ]; + + microvm.vms.k3s-test-1-master-2 = { + autostart = true; + restartIfChanged = true; + specialArgs = { + inherit + myvars + mylib + agenix + mysecrets + ; + }; + config.imports = [ ../k3s-test-1-master-2 ]; + }; + + # Attach the guest's tap to the VM bridge, the same way the physical NIC is + # attached. The tap name is derived from the guest IP (192.168.5.115 -> vm115). + systemd.network.networks."20-vm115" = { + matchConfig.Name = [ "vm115" ]; + networkConfig = { + LinkLocalAddressing = "no"; + Bridge = "br0"; + }; + linkConfig.RequiredForOnline = "no"; + }; +} diff --git a/outputs/x86_64-linux/src/k3s-test-1-master-1.nix b/outputs/x86_64-linux/src/k3s-test-1-master-1.nix index 6cc44109..1ec23a3b 100644 --- a/outputs/x86_64-linux/src/k3s-test-1-master-1.nix +++ b/outputs/x86_64-linux/src/k3s-test-1-master-1.nix @@ -16,16 +16,14 @@ let ssh-user = "root"; modules = { + # the host dir is the complete config (it pulls in the shared modules) nixos-modules = (map mylib.relativeToRoot [ - # common - "secrets/nixos.nix" - "modules/nixos/server/server.nix" - "modules/nixos/server/kubevirt-hardware-configuration.nix" - # host specific "hosts/k8s/${name}" ]) ++ [ + # this runs as a MicroVM + inputs.microvm.nixosModules.microvm { modules.secrets.server.kubernetes.enable = true; } ]; home-modules = map mylib.relativeToRoot [ @@ -39,6 +37,4 @@ in nixosConfigurations.${name} = mylib.nixosSystem systemArgs; colmena.${name} = mylib.colmenaSystem (systemArgs // { inherit tags ssh-user; }); - - packages.${name} = inputs.self.nixosConfigurations.${name}.config.system.build.images.kubevirt; } diff --git a/outputs/x86_64-linux/src/k3s-test-1-master-2.nix b/outputs/x86_64-linux/src/k3s-test-1-master-2.nix index c97ab9eb..062aed88 100644 --- a/outputs/x86_64-linux/src/k3s-test-1-master-2.nix +++ b/outputs/x86_64-linux/src/k3s-test-1-master-2.nix @@ -16,18 +16,19 @@ let ssh-user = "root"; modules = { + # the host dir is the complete config (it pulls in the shared modules) nixos-modules = (map mylib.relativeToRoot [ - # common - "secrets/nixos.nix" - "modules/nixos/server/server.nix" - "modules/nixos/server/kubevirt-hardware-configuration.nix" - # host specific "hosts/k8s/${name}" ]) ++ [ + # this runs as a MicroVM + inputs.microvm.nixosModules.microvm { modules.secrets.server.kubernetes.enable = true; } ]; + home-modules = map mylib.relativeToRoot [ + "home/hosts/linux/${name}.nix" + ]; }; systemArgs = modules // args; @@ -36,6 +37,4 @@ in nixosConfigurations.${name} = mylib.nixosSystem systemArgs; colmena.${name} = mylib.colmenaSystem (systemArgs // { inherit tags ssh-user; }); - - packages.${name} = inputs.self.nixosConfigurations.${name}.config.system.build.images.kubevirt; } diff --git a/outputs/x86_64-linux/src/k3s-test-1-master-3.nix b/outputs/x86_64-linux/src/k3s-test-1-master-3.nix index d8b119a0..e0ad4c80 100644 --- a/outputs/x86_64-linux/src/k3s-test-1-master-3.nix +++ b/outputs/x86_64-linux/src/k3s-test-1-master-3.nix @@ -16,18 +16,19 @@ let ssh-user = "root"; modules = { + # the host dir is the complete config (it pulls in the shared modules) nixos-modules = (map mylib.relativeToRoot [ - # common - "secrets/nixos.nix" - "modules/nixos/server/server.nix" - "modules/nixos/server/kubevirt-hardware-configuration.nix" - # host specific "hosts/k8s/${name}" ]) ++ [ + # this runs as a MicroVM + inputs.microvm.nixosModules.microvm { modules.secrets.server.kubernetes.enable = true; } ]; + home-modules = map mylib.relativeToRoot [ + "home/hosts/linux/${name}.nix" + ]; }; systemArgs = modules // args; @@ -36,6 +37,4 @@ in nixosConfigurations.${name} = mylib.nixosSystem systemArgs; colmena.${name} = mylib.colmenaSystem (systemArgs // { inherit tags ssh-user; }); - - packages.${name} = inputs.self.nixosConfigurations.${name}.config.system.build.images.kubevirt; }