Merge pull request #289 from ryan4yin/feat/k3s-microvm

feat(microvm): run the k3s-test-1 masters as microVMs

.gitignore

@@ -229,3 +229,6 @@ __marimo__/
 
 # Streamlit
 .streamlit/secrets.toml
+
+# the library modules under lib/ must stay tracked
+!lib/*.nix

AGENTS.md

@@ -68,6 +68,10 @@ deployments. Keep repository guidance here; reusable global rules live in `agent
 - `just clean`, `just gc`, `just ggc`, and `just game` remove history or amend commits; they are not
   validation steps and require explicit authorization for their target and scope.
 - Do not use `just penvof` for process inspection: it can expose secret values.
+- MicroVM guests (`k3s-test-1-master-*`) keep their state in
+  `/var/lib/microvms/<name>/{etc,var,home}.img` on their host. Deleting or recreating an image loses
+  the node's identity and cluster state; to resize one, stop the guest and grow the image in place
+  (`truncate` + `e2fsck -f` + `resize2fs`) rather than re-creating it.
 
 ## Further Context
 

Virtual-Machine.md

@@ -1,5 +1,9 @@
 ## How to create & managage KubeVirt's Virtual Machine from this flake?
 
+> [!NOTE] KubeVirt is being retired. aquamarine runs natively on `kubevirt-youko` and the k3s-test
+> nodes run as [microVMs](./hosts/k8s/README.md); this document only covers the KubeVirt VMs that
+> are left to move (currently the Windows VM).
+
 Use `aquamarine` as an example, first build and upload the virtual machine's qcow2 image to the file
 server:
 
@@ -19,7 +23,6 @@ commands:
 ```shell
 just col aquamarine
 just col kubevirt-shoryu
-just col k3s-test-1-master-1
 
 # Set a configuration for the next boot instead of switching immediately
 just col aquamarine boot

hosts/k8s/README.md

@@ -17,26 +17,34 @@ cluster, including other Kubernetes clusters.
 
 ## K3s Clusters
 
-Clusters running as virtual machines on the KubeVirt cluster, for testing and development.
+The control plane runs as NixOS **microVMs** ([microvm.nix]) on the physical hosts, not as KubeVirt
+VMs. Each guest keeps its state in three sparse ext4 images under the host's
+`/var/lib/microvms/<name>/`:
 
-![](_img/2024-02-18_k8s-nodes-overview.webp)
+- `etc.img` (64M) — the ssh host key (also the agenix age identity) and `machine-id`
+- `var.img` (20G) — k3s state (`/var/lib/rancher/k3s`) and the NixOS uid/gid maps
+- `home.img` (4G) — the user's home
 
-1. `k3s-test-1-master-1`
-1. `k3s-test-1-master-2`
-1. `k3s-test-1-master-3`
-1. `k3s-test-1-worker-1`
-1. `k3s-test-1-worker-2`
-1. `k3s-test-1-worker-3`
+The guest's root is a tmpfs and `/nix/store` is the host's store shared read-only (virtiofs), so no
+image has to be built or uploaded; only the volumes above persist. The guest's tap interface is
+bridged onto `br0`, and its name is derived from the guest IP (e.g. `192.168.5.114` -> `vm114`, as
+`IFNAMSIZ` caps interface names at 15 characters).
+
+To move an existing KubeVirt VM over in place, copy the host key and `machine-id` from its disk into
+`etc.img`, and `/var/lib/rancher/k3s/server` into `var.img`, **before the first boot** — then it
+keeps its identity and etcd membership instead of bootstraping a new cluster.
+
+1. `k3s-test-1-master-{1,2,3}` — running as microVMs
+1. `k3s-test-1-worker-{1,2,3}` — to be re-added as microVMs
 
 ## TODO / Known issues
 
+- **Retire KubeVirt.** aquamarine now runs natively on `kubevirt-youko`, and the k3s-test masters
+  run as microVMs. Remove KubeVirt/CDI/multus from the cluster (k8s-gitops) once nothing else needs
+  them, and the Windows VM is the only KubeVirt VM left to move.
 - **The USB HDD bridge is flaky.** aquamarine's two HDDs sit behind a JMicron JMS567 USB-SATA bridge
   that keeps resetting (`dmesg` on `kubevirt-youko`). Find out how often it resets and how much it
   matters before putting anything critical (e.g. an NFS export) on it.
-- **Move aquamarine out of KubeVirt.** It is a "core" machine, so it should not depend on the
-  k8s/KubeVirt control plane; run it as a NixOS microVM on `kubevirt-youko` (`microvm.nix`) instead,
-  reusing its current disk. Consider consolidating the NFS server onto it afterwards (with a fast
-  and a slow tier).
 
 ## Kubernetes Resources
 
@@ -44,5 +52,6 @@ Kubernetes resources are deployed and managed separately through
 [ryan4yin/k8s-gitops](https://github.com/ryan4yin/k8s-gitops).
 
 [k3s]: https://github.com/k3s-io/k3s/
+[microvm.nix]: https://github.com/microvm-nix/microvm.nix
 [what-have-k3s-removed-from-upstream-kubernetes]:
   https://github.com/k3s-io/k3s/?tab=readme-ov-file#what-have-you-removed-from-upstream-kubernetes

hosts/k8s/k3s-test-1-master-1/default.nix

@@ -1,6 +1,7 @@
 {
   config,
   pkgs,
+  lib,
   myvars,
   mylib,
   ...
@@ -8,9 +9,12 @@
 let
   hostName = "k3s-test-1-master-1"; # Define your hostname.
 
-  coreModule = mylib.genKubeVirtGuestModule {
+  coreModule = mylib.genMicrovmGuestModule {
     inherit pkgs hostName;
     inherit (myvars) networking;
+    vcpu = 2;
+    mem = 8192;
+    varSize = 20480;
   };
   k3sModule = mylib.genK3sServerModule {
     inherit pkgs;
@@ -35,8 +39,22 @@ let
   };
 in
 {
-  imports = (mylib.scanPaths ./.) ++ [
-    coreModule
-    k3sModule
-  ];
+  imports =
+    (mylib.scanPaths ./.)
+    ++ (map mylib.relativeToRoot [
+      "secrets/nixos.nix"
+      "modules/nixos/server/server.nix"
+    ])
+    ++ [
+      coreModule
+      k3sModule
+    ];
+
+  # k3s-test-1-token lives in this secret group
+  modules.secrets.server.kubernetes.enable = true;
+
+  # The guest reuses the host's already-instantiated nixpkgs (shared store),
+  # so nixpkgs.config may not be set. The shared base modules set
+  # nixpkgs.config.allowUnfree, so force it empty here.
+  nixpkgs.config = lib.mkForce { };
 }

hosts/k8s/k3s-test-1-master-2/default.nix

@@ -1,16 +1,20 @@
 {
   config,
   pkgs,
+  lib,
   myvars,
   mylib,
   ...
 }:
 let
-  hostName = "k3s-test-1-master-2"; # define your hostname.
+  hostName = "k3s-test-1-master-2"; # Define your hostname.
 
-  coreModule = mylib.genKubeVirtGuestModule {
+  coreModule = mylib.genMicrovmGuestModule {
     inherit pkgs hostName;
     inherit (myvars) networking;
+    vcpu = 2;
+    mem = 4096;
+    varSize = 20480;
   };
   k3sModule = mylib.genK3sServerModule {
     inherit pkgs;
@@ -22,19 +26,25 @@ let
     # the control plane only runs the control plane; workloads run on the
     # k3s-test-1-worker-* nodes (labelled node-role.kubernetes.io/worker=true)
     nodeTaints = [ "node-role.kubernetes.io/control-plane:NoSchedule" ];
-
-    # k3sExtraArgs = [
-    #   # IPv4 Private CIDR(full) - 172.16.0.0/12
-    #   # IPv4 Pod     CIDR(full) - fdfd:cafe:00:0000::/64 ~ fdfd:cafe:00:7fff::/64
-    #   # IPv4 Service CIDR(full) - fdfd:cafe:00:8000::/64 ~ fdfd:cafe:00:ffff::/64
-    #   "--cluster-cidr=172.18.0.0/16,fdfd:cafe:00:0002::/64"
-    #   "--service-cidr=172.19.0.0/16,fdfd:cafe:00:8002::/112"
-    # ];
   };
 in
 {
-  imports = (mylib.scanPaths ./.) ++ [
-    coreModule
-    k3sModule
-  ];
+  imports =
+    (mylib.scanPaths ./.)
+    ++ (map mylib.relativeToRoot [
+      "secrets/nixos.nix"
+      "modules/nixos/server/server.nix"
+    ])
+    ++ [
+      coreModule
+      k3sModule
+    ];
+
+  # k3s-test-1-token lives in this secret group
+  modules.secrets.server.kubernetes.enable = true;
+
+  # The guest reuses the host's already-instantiated nixpkgs (shared store),
+  # so nixpkgs.config may not be set. The shared base modules set
+  # nixpkgs.config.allowUnfree, so force it empty here.
+  nixpkgs.config = lib.mkForce { };
 }

hosts/k8s/k3s-test-1-master-3/default.nix

@@ -1,16 +1,20 @@
 {
   config,
   pkgs,
+  lib,
   myvars,
   mylib,
   ...
 }:
 let
-  hostName = "k3s-test-1-master-3"; # define your hostname.
+  hostName = "k3s-test-1-master-3"; # Define your hostname.
 
-  coreModule = mylib.genKubeVirtGuestModule {
+  coreModule = mylib.genMicrovmGuestModule {
     inherit pkgs hostName;
     inherit (myvars) networking;
+    vcpu = 2;
+    mem = 8192;
+    varSize = 20480;
   };
   k3sModule = mylib.genK3sServerModule {
     inherit pkgs;
@@ -22,19 +26,25 @@ let
     # the control plane only runs the control plane; workloads run on the
     # k3s-test-1-worker-* nodes (labelled node-role.kubernetes.io/worker=true)
     nodeTaints = [ "node-role.kubernetes.io/control-plane:NoSchedule" ];
-
-    # k3sExtraArgs = [
-    #   # IPv4 Private CIDR(full) - 172.16.0.0/12
-    #   # IPv4 Pod     CIDR(full) - fdfd:cafe:00:0000::/64 ~ fdfd:cafe:00:7fff::/64
-    #   # IPv4 Service CIDR(full) - fdfd:cafe:00:8000::/64 ~ fdfd:cafe:00:ffff::/64
-    #   "--cluster-cidr=172.18.0.0/16,fdfd:cafe:00:0002::/64"
-    #   "--service-cidr=172.19.0.0/16,fdfd:cafe:00:8002::/112"
-    # ];
   };
 in
 {
-  imports = (mylib.scanPaths ./.) ++ [
-    coreModule
-    k3sModule
-  ];
+  imports =
+    (mylib.scanPaths ./.)
+    ++ (map mylib.relativeToRoot [
+      "secrets/nixos.nix"
+      "modules/nixos/server/server.nix"
+    ])
+    ++ [
+      coreModule
+      k3sModule
+    ];
+
+  # k3s-test-1-token lives in this secret group
+  modules.secrets.server.kubernetes.enable = true;
+
+  # The guest reuses the host's already-instantiated nixpkgs (shared store),
+  # so nixpkgs.config may not be set. The shared base modules set
+  # nixpkgs.config.allowUnfree, so force it empty here.
+  nixpkgs.config = lib.mkForce { };
 }

hosts/k8s/kubevirt-shoryu/microvm.nix

@@ -0,0 +1,37 @@
+{
+  microvm,
+  myvars,
+  mylib,
+  agenix,
+  mysecrets,
+  ...
+}:
+{
+  # Run the k3s-test VMs as MicroVMs instead of KubeVirt domains.
+  imports = [ microvm.nixosModules.host ];
+
+  microvm.vms.k3s-test-1-master-1 = {
+    autostart = true;
+    restartIfChanged = true;
+    specialArgs = {
+      inherit
+        myvars
+        mylib
+        agenix
+        mysecrets
+        ;
+    };
+    config.imports = [ ../k3s-test-1-master-1 ];
+  };
+
+  # Attach the guest's tap to the VM bridge, the same way the physical NIC is
+  # attached. The tap name is derived from the guest IP (192.168.5.114 -> vm114).
+  systemd.network.networks."20-vm114" = {
+    matchConfig.Name = [ "vm114" ];
+    networkConfig = {
+      LinkLocalAddressing = "no";
+      Bridge = "br0";
+    };
+    linkConfig.RequiredForOnline = "no";
+  };
+}

hosts/k8s/kubevirt-shushou/microvm.nix

@@ -0,0 +1,37 @@
+{
+  microvm,
+  myvars,
+  mylib,
+  agenix,
+  mysecrets,
+  ...
+}:
+{
+  # Run the k3s-test VMs as MicroVMs instead of KubeVirt domains.
+  imports = [ microvm.nixosModules.host ];
+
+  microvm.vms.k3s-test-1-master-3 = {
+    autostart = true;
+    restartIfChanged = true;
+    specialArgs = {
+      inherit
+        myvars
+        mylib
+        agenix
+        mysecrets
+        ;
+    };
+    config.imports = [ ../k3s-test-1-master-3 ];
+  };
+
+  # Attach the guest's tap to the VM bridge, the same way the physical NIC is
+  # attached. The tap name is derived from the guest IP (192.168.5.116 -> vm116).
+  systemd.network.networks."20-vm116" = {
+    matchConfig.Name = [ "vm116" ];
+    networkConfig = {
+      LinkLocalAddressing = "no";
+      Bridge = "br0";
+    };
+    linkConfig.RequiredForOnline = "no";
+  };
+}

hosts/k8s/kubevirt-youko/microvm.nix

@@ -0,0 +1,37 @@
+{
+  microvm,
+  myvars,
+  mylib,
+  agenix,
+  mysecrets,
+  ...
+}:
+{
+  # Run the k3s-test VMs as MicroVMs instead of KubeVirt domains.
+  imports = [ microvm.nixosModules.host ];
+
+  microvm.vms.k3s-test-1-master-2 = {
+    autostart = true;
+    restartIfChanged = true;
+    specialArgs = {
+      inherit
+        myvars
+        mylib
+        agenix
+        mysecrets
+        ;
+    };
+    config.imports = [ ../k3s-test-1-master-2 ];
+  };
+
+  # Attach the guest's tap to the VM bridge, the same way the physical NIC is
+  # attached. The tap name is derived from the guest IP (192.168.5.115 -> vm115).
+  systemd.network.networks."20-vm115" = {
+    matchConfig.Name = [ "vm115" ];
+    networkConfig = {
+      LinkLocalAddressing = "no";
+      Bridge = "br0";
+    };
+    linkConfig.RequiredForOnline = "no";
+  };
+}

lib/default.nix

@@ -10,6 +10,7 @@
   genK3sAgentModule = import ./genK3sAgentModule.nix;
   genKubeVirtHostModule = import ./genKubeVirtHostModule.nix;
   genKubeVirtGuestModule = import ./genKubeVirtGuestModule.nix;
+  genMicrovmGuestModule = import ./genMicrovmGuestModule.nix;
 
   # use path relative to the root of the project
   relativeToRoot = lib.path.append ../.;

lib/genMicrovmGuestModule.nix

@@ -0,0 +1,122 @@
+{
+  pkgs,
+  hostName,
+  networking,
+  # vCPU count and RAM (MiB) -- mirrors the KubeVirt instancetype
+  vcpu,
+  mem,
+  # persistent volume sizes in MiB
+  etcSize ? 64,
+  varSize ? 4096,
+  homeSize ? 4096,
+  # extra stateful mounts (e.g. aquamarine's /data on the passed-through HDDs)
+  extraVolumes ? [ ],
+  ...
+}:
+let
+  inherit (networking) proxyGateway proxyGateway6;
+  inherit (networking.hostsAddr.${hostName}) ipv4;
+  ipv4WithMask = "${ipv4}/24";
+
+  # deterministic MAC per guest so the tap is stable and the guest can match its
+  # NIC by MAC (the qemu NIC name inside the guest is not enp2s0)
+  octets = builtins.match "([0-9]+)\\.([0-9]+)\\.([0-9]+)\\.([0-9]+)" ipv4;
+  toHex =
+    n:
+    let
+      h = pkgs.lib.toHexString (builtins.fromJSON n);
+    in
+    if builtins.stringLength h == 1 then "0${h}" else h;
+  mac = "02:00:00:00:${toHex (builtins.elemAt octets 2)}:${toHex (builtins.elemAt octets 3)}";
+
+  # tap iface name on the host (IFNAMSIZ is 16, so <= 15 chars). All homelab VMs
+  # are 192.168.5.0/24, so the last octet is unique per host.
+  tapId = "vm${builtins.elemAt octets 3}";
+in
+{
+  # The guest is a MicroVM: the root is in RAM, /nix/store is the host's
+  # (read-only, via virtiofs) so no OS image has to be built or uploaded --
+  # everything here comes from this flake. Only the *stateful* paths below are
+  # on real volumes and therefore survive a reboot.
+  microvm = {
+    hypervisor = "qemu";
+    inherit vcpu mem;
+    socket = "control.socket";
+
+    interfaces = [
+      {
+        type = "tap";
+        id = tapId;
+        inherit mac;
+      }
+    ];
+
+    shares = [
+      {
+        tag = "ro-store";
+        proto = "virtiofs";
+        source = "/nix/store";
+        mountPoint = "/nix/.ro-store";
+      }
+    ];
+
+    volumes = [
+      {
+        mountPoint = "/etc"; # ssh host keys (the agenix age identity!), machine-id
+        image = "etc.img";
+        size = etcSize;
+      }
+      {
+        mountPoint = "/var"; # k3s data (/var/lib/rancher/k3s), nixos uid/gid maps
+        image = "var.img";
+        size = varSize;
+      }
+      {
+        mountPoint = "/home"; # user data
+        image = "home.img";
+        size = homeSize;
+      }
+    ]
+    ++ extraVolumes;
+  };
+
+  networking = {
+    inherit hostName;
+    networkmanager.enable = false;
+    useDHCP = false;
+  };
+  networking.useNetworkd = true;
+  systemd.network.enable = true;
+
+  # match the virtio NIC by MAC: qemu names it something like enp0s1, not enp2s0
+  systemd.network.networks."10-${hostName}" = {
+    matchConfig.MACAddress = [ mac ];
+    networkConfig = {
+      Address = [ ipv4WithMask ];
+      DNS = [ proxyGateway ];
+      DHCP = "ipv6";
+      IPv6AcceptRA = true;
+      LinkLocalAddressing = "ipv6";
+    };
+    routes = [
+      {
+        Destination = "0.0.0.0/0";
+        Gateway = proxyGateway;
+      }
+      {
+        Destination = "::/0";
+        Gateway = proxyGateway6;
+        GatewayOnLink = true;
+      }
+    ];
+    linkConfig.RequiredForOnline = "routable";
+  };
+
+  # journald on tmpfs -- keep it small
+  services.journald.settings.Journal = {
+    SystemMaxUse = "512M";
+    RuntimeMaxUse = "256M";
+  };
+
+  system.stateVersion = "26.05";
+}

outputs/x86_64-linux/src/k3s-test-1-master-1.nix

@@ -16,16 +16,14 @@ let
   ssh-user = "root";
 
   modules = {
+    # the host dir is the complete config (it pulls in the shared modules)
     nixos-modules =
       (map mylib.relativeToRoot [
-        # common
-        "secrets/nixos.nix"
-        "modules/nixos/server/server.nix"
-        "modules/nixos/server/kubevirt-hardware-configuration.nix"
-        # host specific
         "hosts/k8s/${name}"
       ])
       ++ [
+        # this runs as a MicroVM
+        inputs.microvm.nixosModules.microvm
         { modules.secrets.server.kubernetes.enable = true; }
       ];
     home-modules = map mylib.relativeToRoot [
@@ -39,6 +37,4 @@ in
   nixosConfigurations.${name} = mylib.nixosSystem systemArgs;
 
   colmena.${name} = mylib.colmenaSystem (systemArgs // { inherit tags ssh-user; });
-
-  packages.${name} = inputs.self.nixosConfigurations.${name}.config.system.build.images.kubevirt;
 }

outputs/x86_64-linux/src/k3s-test-1-master-2.nix

@@ -16,18 +16,19 @@ let
   ssh-user = "root";
 
   modules = {
+    # the host dir is the complete config (it pulls in the shared modules)
     nixos-modules =
       (map mylib.relativeToRoot [
-        # common
-        "secrets/nixos.nix"
-        "modules/nixos/server/server.nix"
-        "modules/nixos/server/kubevirt-hardware-configuration.nix"
-        # host specific
         "hosts/k8s/${name}"
       ])
       ++ [
+        # this runs as a MicroVM
+        inputs.microvm.nixosModules.microvm
         { modules.secrets.server.kubernetes.enable = true; }
       ];
+    home-modules = map mylib.relativeToRoot [
+      "home/hosts/linux/${name}.nix"
+    ];
   };
 
   systemArgs = modules // args;
@@ -36,6 +37,4 @@ in
   nixosConfigurations.${name} = mylib.nixosSystem systemArgs;
 
   colmena.${name} = mylib.colmenaSystem (systemArgs // { inherit tags ssh-user; });
-
-  packages.${name} = inputs.self.nixosConfigurations.${name}.config.system.build.images.kubevirt;
 }

outputs/x86_64-linux/src/k3s-test-1-master-3.nix

@@ -16,18 +16,19 @@ let
   ssh-user = "root";
 
   modules = {
+    # the host dir is the complete config (it pulls in the shared modules)
     nixos-modules =
       (map mylib.relativeToRoot [
-        # common
-        "secrets/nixos.nix"
-        "modules/nixos/server/server.nix"
-        "modules/nixos/server/kubevirt-hardware-configuration.nix"
-        # host specific
         "hosts/k8s/${name}"
       ])
       ++ [
+        # this runs as a MicroVM
+        inputs.microvm.nixosModules.microvm
         { modules.secrets.server.kubernetes.enable = true; }
       ];
+    home-modules = map mylib.relativeToRoot [
+      "home/hosts/linux/${name}.nix"
+    ];
   };
 
   systemArgs = modules // args;
@@ -36,6 +37,4 @@ in
   nixosConfigurations.${name} = mylib.nixosSystem systemArgs;
 
   colmena.${name} = mylib.colmenaSystem (systemArgs // { inherit tags ssh-user; });
-
-  packages.${name} = inputs.self.nixosConfigurations.${name}.config.system.build.images.kubevirt;
 }