diff --git a/.gitignore b/.gitignore index b5e95bc1..0083f18e 100644 --- a/.gitignore +++ b/.gitignore @@ -229,3 +229,6 @@ __marimo__/ # Streamlit .streamlit/secrets.toml + +# the library modules under lib/ must stay tracked +!lib/*.nix diff --git a/AGENTS.md b/AGENTS.md index 369b842e..639f25e3 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -68,6 +68,10 @@ deployments. Keep repository guidance here; reusable global rules live in `agent - `just clean`, `just gc`, `just ggc`, and `just game` remove history or amend commits; they are not validation steps and require explicit authorization for their target and scope. - Do not use `just penvof` for process inspection: it can expose secret values. +- MicroVM guests (`k3s-test-1-master-*`) keep their state in + `/var/lib/microvms//{etc,var,home}.img` on their host. Deleting or recreating an image loses + the node's identity and cluster state; to resize one, stop the guest and grow the image in place + (`truncate` + `e2fsck -f` + `resize2fs`) rather than re-creating it. ## Further Context diff --git a/Virtual-Machine.md b/Virtual-Machine.md index 8bae5e46..c0d31c1c 100644 --- a/Virtual-Machine.md +++ b/Virtual-Machine.md @@ -1,5 +1,9 @@ ## How to create & managage KubeVirt's Virtual Machine from this flake? +> [!NOTE] KubeVirt is being retired. aquamarine runs natively on `kubevirt-youko` and the k3s-test +> nodes run as [microVMs](./hosts/k8s/README.md); this document only covers the KubeVirt VMs that +> are left to move (currently the Windows VM). + Use `aquamarine` as an example, first build and upload the virtual machine's qcow2 image to the file server: @@ -19,7 +23,6 @@ commands: ```shell just col aquamarine just col kubevirt-shoryu -just col k3s-test-1-master-1 # Set a configuration for the next boot instead of switching immediately just col aquamarine boot diff --git a/hosts/k8s/README.md b/hosts/k8s/README.md index 6dfe3089..64e26d1d 100644 --- a/hosts/k8s/README.md +++ b/hosts/k8s/README.md @@ -17,26 +17,34 @@ cluster, including other Kubernetes clusters. ## K3s Clusters -Clusters running as virtual machines on the KubeVirt cluster, for testing and development. +The control plane runs as NixOS **microVMs** ([microvm.nix]) on the physical hosts, not as KubeVirt +VMs. Each guest keeps its state in three sparse ext4 images under the host's +`/var/lib/microvms//`: -![](_img/2024-02-18_k8s-nodes-overview.webp) +- `etc.img` (64M) — the ssh host key (also the agenix age identity) and `machine-id` +- `var.img` (20G) — k3s state (`/var/lib/rancher/k3s`) and the NixOS uid/gid maps +- `home.img` (4G) — the user's home -1. `k3s-test-1-master-1` -1. `k3s-test-1-master-2` -1. `k3s-test-1-master-3` -1. `k3s-test-1-worker-1` -1. `k3s-test-1-worker-2` -1. `k3s-test-1-worker-3` +The guest's root is a tmpfs and `/nix/store` is the host's store shared read-only (virtiofs), so no +image has to be built or uploaded; only the volumes above persist. The guest's tap interface is +bridged onto `br0`, and its name is derived from the guest IP (e.g. `192.168.5.114` -> `vm114`, as +`IFNAMSIZ` caps interface names at 15 characters). + +To move an existing KubeVirt VM over in place, copy the host key and `machine-id` from its disk into +`etc.img`, and `/var/lib/rancher/k3s/server` into `var.img`, **before the first boot** — then it +keeps its identity and etcd membership instead of bootstraping a new cluster. + +1. `k3s-test-1-master-{1,2,3}` — running as microVMs +1. `k3s-test-1-worker-{1,2,3}` — to be re-added as microVMs ## TODO / Known issues +- **Retire KubeVirt.** aquamarine now runs natively on `kubevirt-youko`, and the k3s-test masters + run as microVMs. Remove KubeVirt/CDI/multus from the cluster (k8s-gitops) once nothing else needs + them, and the Windows VM is the only KubeVirt VM left to move. - **The USB HDD bridge is flaky.** aquamarine's two HDDs sit behind a JMicron JMS567 USB-SATA bridge that keeps resetting (`dmesg` on `kubevirt-youko`). Find out how often it resets and how much it matters before putting anything critical (e.g. an NFS export) on it. -- **Move aquamarine out of KubeVirt.** It is a "core" machine, so it should not depend on the - k8s/KubeVirt control plane; run it as a NixOS microVM on `kubevirt-youko` (`microvm.nix`) instead, - reusing its current disk. Consider consolidating the NFS server onto it afterwards (with a fast - and a slow tier). ## Kubernetes Resources @@ -44,5 +52,6 @@ Kubernetes resources are deployed and managed separately through [ryan4yin/k8s-gitops](https://github.com/ryan4yin/k8s-gitops). [k3s]: https://github.com/k3s-io/k3s/ +[microvm.nix]: https://github.com/microvm-nix/microvm.nix [what-have-k3s-removed-from-upstream-kubernetes]: https://github.com/k3s-io/k3s/?tab=readme-ov-file#what-have-you-removed-from-upstream-kubernetes diff --git a/hosts/k8s/k3s-test-1-master-1/default.nix b/hosts/k8s/k3s-test-1-master-1/default.nix index f9a5f87b..6a42651f 100644 --- a/hosts/k8s/k3s-test-1-master-1/default.nix +++ b/hosts/k8s/k3s-test-1-master-1/default.nix @@ -1,6 +1,7 @@ { config, pkgs, + lib, myvars, mylib, ... @@ -8,9 +9,12 @@ let hostName = "k3s-test-1-master-1"; # Define your hostname. - coreModule = mylib.genKubeVirtGuestModule { + coreModule = mylib.genMicrovmGuestModule { inherit pkgs hostName; inherit (myvars) networking; + vcpu = 2; + mem = 8192; + varSize = 20480; }; k3sModule = mylib.genK3sServerModule { inherit pkgs; @@ -35,8 +39,22 @@ let }; in { - imports = (mylib.scanPaths ./.) ++ [ - coreModule - k3sModule - ]; + imports = + (mylib.scanPaths ./.) + ++ (map mylib.relativeToRoot [ + "secrets/nixos.nix" + "modules/nixos/server/server.nix" + ]) + ++ [ + coreModule + k3sModule + ]; + + # k3s-test-1-token lives in this secret group + modules.secrets.server.kubernetes.enable = true; + + # The guest reuses the host's already-instantiated nixpkgs (shared store), + # so nixpkgs.config may not be set. The shared base modules set + # nixpkgs.config.allowUnfree, so force it empty here. + nixpkgs.config = lib.mkForce { }; } diff --git a/hosts/k8s/k3s-test-1-master-2/default.nix b/hosts/k8s/k3s-test-1-master-2/default.nix index 8f60aa7d..7c5f1200 100644 --- a/hosts/k8s/k3s-test-1-master-2/default.nix +++ b/hosts/k8s/k3s-test-1-master-2/default.nix @@ -1,16 +1,20 @@ { config, pkgs, + lib, myvars, mylib, ... }: let - hostName = "k3s-test-1-master-2"; # define your hostname. + hostName = "k3s-test-1-master-2"; # Define your hostname. - coreModule = mylib.genKubeVirtGuestModule { + coreModule = mylib.genMicrovmGuestModule { inherit pkgs hostName; inherit (myvars) networking; + vcpu = 2; + mem = 4096; + varSize = 20480; }; k3sModule = mylib.genK3sServerModule { inherit pkgs; @@ -22,19 +26,25 @@ let # the control plane only runs the control plane; workloads run on the # k3s-test-1-worker-* nodes (labelled node-role.kubernetes.io/worker=true) nodeTaints = [ "node-role.kubernetes.io/control-plane:NoSchedule" ]; - - # k3sExtraArgs = [ - # # IPv4 Private CIDR(full) - 172.16.0.0/12 - # # IPv4 Pod CIDR(full) - fdfd:cafe:00:0000::/64 ~ fdfd:cafe:00:7fff::/64 - # # IPv4 Service CIDR(full) - fdfd:cafe:00:8000::/64 ~ fdfd:cafe:00:ffff::/64 - # "--cluster-cidr=172.18.0.0/16,fdfd:cafe:00:0002::/64" - # "--service-cidr=172.19.0.0/16,fdfd:cafe:00:8002::/112" - # ]; }; in { - imports = (mylib.scanPaths ./.) ++ [ - coreModule - k3sModule - ]; + imports = + (mylib.scanPaths ./.) + ++ (map mylib.relativeToRoot [ + "secrets/nixos.nix" + "modules/nixos/server/server.nix" + ]) + ++ [ + coreModule + k3sModule + ]; + + # k3s-test-1-token lives in this secret group + modules.secrets.server.kubernetes.enable = true; + + # The guest reuses the host's already-instantiated nixpkgs (shared store), + # so nixpkgs.config may not be set. The shared base modules set + # nixpkgs.config.allowUnfree, so force it empty here. + nixpkgs.config = lib.mkForce { }; } diff --git a/hosts/k8s/k3s-test-1-master-3/default.nix b/hosts/k8s/k3s-test-1-master-3/default.nix index d11e1e7b..8d2fd2e0 100644 --- a/hosts/k8s/k3s-test-1-master-3/default.nix +++ b/hosts/k8s/k3s-test-1-master-3/default.nix @@ -1,16 +1,20 @@ { config, pkgs, + lib, myvars, mylib, ... }: let - hostName = "k3s-test-1-master-3"; # define your hostname. + hostName = "k3s-test-1-master-3"; # Define your hostname. - coreModule = mylib.genKubeVirtGuestModule { + coreModule = mylib.genMicrovmGuestModule { inherit pkgs hostName; inherit (myvars) networking; + vcpu = 2; + mem = 8192; + varSize = 20480; }; k3sModule = mylib.genK3sServerModule { inherit pkgs; @@ -22,19 +26,25 @@ let # the control plane only runs the control plane; workloads run on the # k3s-test-1-worker-* nodes (labelled node-role.kubernetes.io/worker=true) nodeTaints = [ "node-role.kubernetes.io/control-plane:NoSchedule" ]; - - # k3sExtraArgs = [ - # # IPv4 Private CIDR(full) - 172.16.0.0/12 - # # IPv4 Pod CIDR(full) - fdfd:cafe:00:0000::/64 ~ fdfd:cafe:00:7fff::/64 - # # IPv4 Service CIDR(full) - fdfd:cafe:00:8000::/64 ~ fdfd:cafe:00:ffff::/64 - # "--cluster-cidr=172.18.0.0/16,fdfd:cafe:00:0002::/64" - # "--service-cidr=172.19.0.0/16,fdfd:cafe:00:8002::/112" - # ]; }; in { - imports = (mylib.scanPaths ./.) ++ [ - coreModule - k3sModule - ]; + imports = + (mylib.scanPaths ./.) + ++ (map mylib.relativeToRoot [ + "secrets/nixos.nix" + "modules/nixos/server/server.nix" + ]) + ++ [ + coreModule + k3sModule + ]; + + # k3s-test-1-token lives in this secret group + modules.secrets.server.kubernetes.enable = true; + + # The guest reuses the host's already-instantiated nixpkgs (shared store), + # so nixpkgs.config may not be set. The shared base modules set + # nixpkgs.config.allowUnfree, so force it empty here. + nixpkgs.config = lib.mkForce { }; } diff --git a/hosts/k8s/kubevirt-shoryu/microvm.nix b/hosts/k8s/kubevirt-shoryu/microvm.nix new file mode 100644 index 00000000..fca62fb8 --- /dev/null +++ b/hosts/k8s/kubevirt-shoryu/microvm.nix @@ -0,0 +1,37 @@ +{ + microvm, + myvars, + mylib, + agenix, + mysecrets, + ... +}: +{ + # Run the k3s-test VMs as MicroVMs instead of KubeVirt domains. + imports = [ microvm.nixosModules.host ]; + + microvm.vms.k3s-test-1-master-1 = { + autostart = true; + restartIfChanged = true; + specialArgs = { + inherit + myvars + mylib + agenix + mysecrets + ; + }; + config.imports = [ ../k3s-test-1-master-1 ]; + }; + + # Attach the guest's tap to the VM bridge, the same way the physical NIC is + # attached. The tap name is derived from the guest IP (192.168.5.114 -> vm114). + systemd.network.networks."20-vm114" = { + matchConfig.Name = [ "vm114" ]; + networkConfig = { + LinkLocalAddressing = "no"; + Bridge = "br0"; + }; + linkConfig.RequiredForOnline = "no"; + }; +} diff --git a/hosts/k8s/kubevirt-shushou/microvm.nix b/hosts/k8s/kubevirt-shushou/microvm.nix new file mode 100644 index 00000000..72871125 --- /dev/null +++ b/hosts/k8s/kubevirt-shushou/microvm.nix @@ -0,0 +1,37 @@ +{ + microvm, + myvars, + mylib, + agenix, + mysecrets, + ... +}: +{ + # Run the k3s-test VMs as MicroVMs instead of KubeVirt domains. + imports = [ microvm.nixosModules.host ]; + + microvm.vms.k3s-test-1-master-3 = { + autostart = true; + restartIfChanged = true; + specialArgs = { + inherit + myvars + mylib + agenix + mysecrets + ; + }; + config.imports = [ ../k3s-test-1-master-3 ]; + }; + + # Attach the guest's tap to the VM bridge, the same way the physical NIC is + # attached. The tap name is derived from the guest IP (192.168.5.116 -> vm116). + systemd.network.networks."20-vm116" = { + matchConfig.Name = [ "vm116" ]; + networkConfig = { + LinkLocalAddressing = "no"; + Bridge = "br0"; + }; + linkConfig.RequiredForOnline = "no"; + }; +} diff --git a/hosts/k8s/kubevirt-youko/microvm.nix b/hosts/k8s/kubevirt-youko/microvm.nix new file mode 100644 index 00000000..96150b0d --- /dev/null +++ b/hosts/k8s/kubevirt-youko/microvm.nix @@ -0,0 +1,37 @@ +{ + microvm, + myvars, + mylib, + agenix, + mysecrets, + ... +}: +{ + # Run the k3s-test VMs as MicroVMs instead of KubeVirt domains. + imports = [ microvm.nixosModules.host ]; + + microvm.vms.k3s-test-1-master-2 = { + autostart = true; + restartIfChanged = true; + specialArgs = { + inherit + myvars + mylib + agenix + mysecrets + ; + }; + config.imports = [ ../k3s-test-1-master-2 ]; + }; + + # Attach the guest's tap to the VM bridge, the same way the physical NIC is + # attached. The tap name is derived from the guest IP (192.168.5.115 -> vm115). + systemd.network.networks."20-vm115" = { + matchConfig.Name = [ "vm115" ]; + networkConfig = { + LinkLocalAddressing = "no"; + Bridge = "br0"; + }; + linkConfig.RequiredForOnline = "no"; + }; +} diff --git a/lib/default.nix b/lib/default.nix index fc9ac40d..3f762523 100644 --- a/lib/default.nix +++ b/lib/default.nix @@ -10,6 +10,7 @@ genK3sAgentModule = import ./genK3sAgentModule.nix; genKubeVirtHostModule = import ./genKubeVirtHostModule.nix; genKubeVirtGuestModule = import ./genKubeVirtGuestModule.nix; + genMicrovmGuestModule = import ./genMicrovmGuestModule.nix; # use path relative to the root of the project relativeToRoot = lib.path.append ../.; diff --git a/lib/genMicrovmGuestModule.nix b/lib/genMicrovmGuestModule.nix new file mode 100644 index 00000000..fea0164c --- /dev/null +++ b/lib/genMicrovmGuestModule.nix @@ -0,0 +1,122 @@ +{ + pkgs, + hostName, + networking, + # vCPU count and RAM (MiB) -- mirrors the KubeVirt instancetype + vcpu, + mem, + # persistent volume sizes in MiB + etcSize ? 64, + varSize ? 4096, + homeSize ? 4096, + # extra stateful mounts (e.g. aquamarine's /data on the passed-through HDDs) + extraVolumes ? [ ], + ... +}: +let + inherit (networking) proxyGateway proxyGateway6; + inherit (networking.hostsAddr.${hostName}) ipv4; + ipv4WithMask = "${ipv4}/24"; + + # deterministic MAC per guest so the tap is stable and the guest can match its + # NIC by MAC (the qemu NIC name inside the guest is not enp2s0) + octets = builtins.match "([0-9]+)\\.([0-9]+)\\.([0-9]+)\\.([0-9]+)" ipv4; + toHex = + n: + let + h = pkgs.lib.toHexString (builtins.fromJSON n); + in + if builtins.stringLength h == 1 then "0${h}" else h; + mac = "02:00:00:00:${toHex (builtins.elemAt octets 2)}:${toHex (builtins.elemAt octets 3)}"; + + # tap iface name on the host (IFNAMSIZ is 16, so <= 15 chars). All homelab VMs + # are 192.168.5.0/24, so the last octet is unique per host. + tapId = "vm${builtins.elemAt octets 3}"; +in +{ + # The guest is a MicroVM: the root is in RAM, /nix/store is the host's + # (read-only, via virtiofs) so no OS image has to be built or uploaded -- + # everything here comes from this flake. Only the *stateful* paths below are + # on real volumes and therefore survive a reboot. + microvm = { + hypervisor = "qemu"; + inherit vcpu mem; + socket = "control.socket"; + + interfaces = [ + { + type = "tap"; + id = tapId; + inherit mac; + } + ]; + + shares = [ + { + tag = "ro-store"; + proto = "virtiofs"; + source = "/nix/store"; + mountPoint = "/nix/.ro-store"; + } + ]; + + volumes = [ + { + mountPoint = "/etc"; # ssh host keys (the agenix age identity!), machine-id + image = "etc.img"; + size = etcSize; + } + { + mountPoint = "/var"; # k3s data (/var/lib/rancher/k3s), nixos uid/gid maps + image = "var.img"; + size = varSize; + } + { + mountPoint = "/home"; # user data + image = "home.img"; + size = homeSize; + } + ] + ++ extraVolumes; + }; + + networking = { + inherit hostName; + networkmanager.enable = false; + useDHCP = false; + }; + networking.useNetworkd = true; + systemd.network.enable = true; + + # match the virtio NIC by MAC: qemu names it something like enp0s1, not enp2s0 + systemd.network.networks."10-${hostName}" = { + matchConfig.MACAddress = [ mac ]; + networkConfig = { + Address = [ ipv4WithMask ]; + DNS = [ proxyGateway ]; + DHCP = "ipv6"; + IPv6AcceptRA = true; + LinkLocalAddressing = "ipv6"; + }; + routes = [ + { + Destination = "0.0.0.0/0"; + Gateway = proxyGateway; + } + { + Destination = "::/0"; + Gateway = proxyGateway6; + GatewayOnLink = true; + } + ]; + linkConfig.RequiredForOnline = "routable"; + }; + + # journald on tmpfs -- keep it small + services.journald.settings.Journal = { + SystemMaxUse = "512M"; + RuntimeMaxUse = "256M"; + }; + + system.stateVersion = "26.05"; +} diff --git a/outputs/x86_64-linux/src/k3s-test-1-master-1.nix b/outputs/x86_64-linux/src/k3s-test-1-master-1.nix index 1c859c04..1ec23a3b 100644 --- a/outputs/x86_64-linux/src/k3s-test-1-master-1.nix +++ b/outputs/x86_64-linux/src/k3s-test-1-master-1.nix @@ -16,16 +16,14 @@ let ssh-user = "root"; modules = { + # the host dir is the complete config (it pulls in the shared modules) nixos-modules = (map mylib.relativeToRoot [ - # common - "secrets/nixos.nix" - "modules/nixos/server/server.nix" - "modules/nixos/server/qemu-guest-hardware-configuration.nix" - # host specific "hosts/k8s/${name}" ]) ++ [ + # this runs as a MicroVM + inputs.microvm.nixosModules.microvm { modules.secrets.server.kubernetes.enable = true; } ]; home-modules = map mylib.relativeToRoot [ @@ -39,6 +37,4 @@ in nixosConfigurations.${name} = mylib.nixosSystem systemArgs; colmena.${name} = mylib.colmenaSystem (systemArgs // { inherit tags ssh-user; }); - - packages.${name} = inputs.self.nixosConfigurations.${name}.config.system.build.images.kubevirt; } diff --git a/outputs/x86_64-linux/src/k3s-test-1-master-2.nix b/outputs/x86_64-linux/src/k3s-test-1-master-2.nix index 6264e370..062aed88 100644 --- a/outputs/x86_64-linux/src/k3s-test-1-master-2.nix +++ b/outputs/x86_64-linux/src/k3s-test-1-master-2.nix @@ -16,18 +16,19 @@ let ssh-user = "root"; modules = { + # the host dir is the complete config (it pulls in the shared modules) nixos-modules = (map mylib.relativeToRoot [ - # common - "secrets/nixos.nix" - "modules/nixos/server/server.nix" - "modules/nixos/server/qemu-guest-hardware-configuration.nix" - # host specific "hosts/k8s/${name}" ]) ++ [ + # this runs as a MicroVM + inputs.microvm.nixosModules.microvm { modules.secrets.server.kubernetes.enable = true; } ]; + home-modules = map mylib.relativeToRoot [ + "home/hosts/linux/${name}.nix" + ]; }; systemArgs = modules // args; @@ -36,6 +37,4 @@ in nixosConfigurations.${name} = mylib.nixosSystem systemArgs; colmena.${name} = mylib.colmenaSystem (systemArgs // { inherit tags ssh-user; }); - - packages.${name} = inputs.self.nixosConfigurations.${name}.config.system.build.images.kubevirt; } diff --git a/outputs/x86_64-linux/src/k3s-test-1-master-3.nix b/outputs/x86_64-linux/src/k3s-test-1-master-3.nix index 496f80d5..e0ad4c80 100644 --- a/outputs/x86_64-linux/src/k3s-test-1-master-3.nix +++ b/outputs/x86_64-linux/src/k3s-test-1-master-3.nix @@ -16,18 +16,19 @@ let ssh-user = "root"; modules = { + # the host dir is the complete config (it pulls in the shared modules) nixos-modules = (map mylib.relativeToRoot [ - # common - "secrets/nixos.nix" - "modules/nixos/server/server.nix" - "modules/nixos/server/qemu-guest-hardware-configuration.nix" - # host specific "hosts/k8s/${name}" ]) ++ [ + # this runs as a MicroVM + inputs.microvm.nixosModules.microvm { modules.secrets.server.kubernetes.enable = true; } ]; + home-modules = map mylib.relativeToRoot [ + "home/hosts/linux/${name}.nix" + ]; }; systemArgs = modules // args; @@ -36,6 +37,4 @@ in nixosConfigurations.${name} = mylib.nixosSystem systemArgs; colmena.${name} = mylib.colmenaSystem (systemArgs // { inherit tags ssh-user; }); - - packages.${name} = inputs.self.nixosConfigurations.${name}.config.system.build.images.kubevirt; }