From 787b51df25d7f4547630213e017e646bddc002b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?ryan4yin=20=7C=20=E4=BA=8C=E8=8A=B1?= Date: Mon, 21 Sep 2026 15:12:37 +0800 Subject: [PATCH] feat(vms): migrate the idol desktops from KubeVirt to libvirt (#287) * wip(kubevirt-shoryu): enable libvirt/qemu * feat(kubevirt-shoryu): enable libvirt/qemu with br0 + no NAT network * chore: lock the microvm flake input * feat(kubevirt-shoryu): define the libvirt domains declaratively * fix(kubevirt-shoryu): pin the kana domain UUID * feat(kubevirt-shoryu): migrate idols-ruby to libvirt --- flake.lock | 38 ++++++++++++ flake.nix | 6 ++ hosts/k8s/kubevirt-shoryu/libvirtd.nix | 82 ++++++++++++++++++++++++++ lib/genLibvirtDomainXml.nix | 59 ++++++++++++++++++ 4 files changed, 185 insertions(+) create mode 100644 hosts/k8s/kubevirt-shoryu/libvirtd.nix create mode 100644 lib/genLibvirtDomainXml.nix diff --git a/flake.lock b/flake.lock index 7f2f18a1..d7d8f9a7 100644 --- a/flake.lock +++ b/flake.lock @@ -462,6 +462,27 @@ "type": "github" } }, + "microvm": { + "inputs": { + "nixpkgs": [ + "nixpkgs" + ], + "spectrum": "spectrum" + }, + "locked": { + "lastModified": 1789506833, + "narHash": "sha256-4L8SOCF84de1irB72Ka9VHcu9wrAJ9Amuey4lU9oh4Y=", + "owner": "microvm-nix", + "repo": "microvm.nix", + "rev": "187b0a390ee054028106a674e7b01b1cb940cbba", + "type": "github" + }, + "original": { + "owner": "microvm-nix", + "repo": "microvm.nix", + "type": "github" + } + }, "my-asahi-firmware": { "flake": false, "locked": { @@ -873,6 +894,7 @@ "home-manager": "home-manager_2", "lanzaboote": "lanzaboote", "llm-agents": "llm-agents", + "microvm": "microvm", "my-asahi-firmware": "my-asahi-firmware", "mysecrets": "mysecrets", "nix-darwin": "nix-darwin", @@ -916,6 +938,22 @@ "type": "github" } }, + "spectrum": { + "flake": false, + "locked": { + "lastModified": 1786032750, + "narHash": "sha256-+Ba/kIn5BjvV7PN8bWOo7RoqgTY/Yv2M7qeMf3wS2Ag=", + "ref": "refs/heads/main", + "rev": "77d2389874bb092b8dbf05ecc5b1030372e80e98", + "revCount": 1417, + "type": "git", + "url": "https://spectrum-os.org/git/spectrum" + }, + "original": { + "type": "git", + "url": "https://spectrum-os.org/git/spectrum" + } + }, "systems": { "locked": { "lastModified": 1681028828, diff --git a/flake.nix b/flake.nix index 578c02e4..1ea51504 100644 --- a/flake.nix +++ b/flake.nix @@ -50,6 +50,12 @@ # get some latest packages from the master branch nixpkgs-master.url = "github:nixos/nixpkgs/master"; + # NixOS MicroVMs -- run the VMs directly on the hosts, replacing KubeVirt + microvm = { + url = "github:microvm-nix/microvm.nix"; + inputs.nixpkgs.follows = "nixpkgs"; + }; + # for macos # nixpkgs-darwin.url = "github:nixos/nixpkgs/nixpkgs-26.05-darwin"; nixpkgs-darwin.url = "github:nixos/nixpkgs/nixpkgs-unstable"; diff --git a/hosts/k8s/kubevirt-shoryu/libvirtd.nix b/hosts/k8s/kubevirt-shoryu/libvirtd.nix new file mode 100644 index 00000000..0a2c15df --- /dev/null +++ b/hosts/k8s/kubevirt-shoryu/libvirtd.nix @@ -0,0 +1,82 @@ +{ + pkgs, + lib, + mylib, + ... +}: +let + genXml = import (mylib.relativeToRoot "lib/genLibvirtDomainXml.nix") { inherit lib; }; + + # Declarative libvirt domains. The disk image is a one-time copy (reflink) of + # the VM's former KubeVirt disk, e.g.: + # cp --reflink=auto /var/lib/rancher/k3s/storage/vms/-disk/disk.img \ + # /var/lib/libvirt/images/.img + domains = { + idols-kana = { + uuid = "e744f563-7c7d-45e6-bc71-61980790208d"; + vcpu = 4; + memoryGiB = 8; + disk = "/var/lib/libvirt/images/idols-kana.img"; + mac = "52:54:00:5a:51:02"; + vncPort = 5902; + }; + idols-ruby = { + uuid = "3a959d02-11de-498d-bce7-85f78713ca63"; + vcpu = 4; + memoryGiB = 8; + disk = "/var/lib/libvirt/images/idols-ruby.img"; + mac = "52:54:00:5a:51:03"; + vncPort = 5903; + }; + }; +in +{ + # libvirt/QEMU -- runs the VMs directly instead of on KubeVirt. Used for the + # VMs that need to feel like a real machine (the agent desktops kana/ruby) and + # for Windows. + virtualisation.libvirtd.enable = true; + + # The VMs attach to our existing bridge (br0), so allow it explicitly. + virtualisation.libvirtd.allowedBridges = [ "br0" ]; + + # We don't want libvirt's NAT network (virbr0): every VM uses br0. + systemd.services.libvirt-disable-default-net = { + description = "Disable libvirt's default NAT network"; + after = [ "libvirtd.service" ]; + requires = [ "libvirtd.service" ]; + wantedBy = [ "multi-user.target" ]; + serviceConfig = { + Type = "oneshot"; + RemainAfterExit = true; + }; + script = '' + ${pkgs.libvirt}/bin/virsh net-destroy default || true + ${pkgs.libvirt}/bin/virsh net-autostart --disable default || true + ''; + }; + + # Define (and autostart) the declarative domains on every activation. + systemd.services.libvirt-define-domains = { + description = "Define the declarative libvirt domains"; + after = [ "libvirtd.service" ]; + requires = [ "libvirtd.service" ]; + wantedBy = [ "multi-user.target" ]; + serviceConfig = { + Type = "oneshot"; + RemainAfterExit = true; + }; + script = lib.concatStringsSep "\n" ( + lib.mapAttrsToList (name: args: '' + ${pkgs.libvirt}/bin/virsh define ${ + pkgs.writeText "libvirt-domain-${name}.xml" (genXml (args // { inherit name; })) + } + ${pkgs.libvirt}/bin/virsh autostart ${name} || true + '') domains + ); + }; + + # VM disk images (the declarative domains reference files here). + systemd.tmpfiles.rules = [ + "d /var/lib/libvirt/images 0755 root root -" + ]; +} diff --git a/lib/genLibvirtDomainXml.nix b/lib/genLibvirtDomainXml.nix new file mode 100644 index 00000000..46001d12 --- /dev/null +++ b/lib/genLibvirtDomainXml.nix @@ -0,0 +1,59 @@ +# Generate a libvirt domain XML for a VM that behaves like a real machine +# (BIOS/q35, virtio disk, bridged NIC, VNC display). +# +# The NIC is pinned to q35 PCI bus 0x02 and the disk to bus 0x04 so the guest +# sees the same interface name it did under KubeVirt (enp2s0) and the same +# device ordering. +{ lib }: +{ + name, + uuid, + vcpu, + memoryGiB, + disk, + mac, + bridge ? "br0", + vncPort, +}: +'' + + ${name} + ${uuid} + ${toString memoryGiB} + ${toString vcpu} + + hvm + + + + + + + destroy + restart + + + + + +
+ + + + + +
+ + + + + + + + /dev/urandom + + + +''