diff --git a/AGENTS.md b/AGENTS.md index 442f9937..369b842e 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -61,10 +61,9 @@ deployments. Keep repository guidance here; reusable global rules live in `agent - Deployment and upload recipes change systems; use eval/build commands for validation. Remote deployment requires an explicit request. When deployment behavior changes, report the exact `just` command to run. -- KubeVirt hosts (`kubevirt-shoryu`/`shushou`/`youko`): avoid activations that restart the OVS - bridge units (`ovs-vswitchd`/`ovsbr1-netdev`) — that recreates the bridge and drops the VM - network. `switch` is fine for changes that don't touch the OVS/network stack; use `boot` + serial - reboot for nixpkgs/Open vSwitch or networking changes. See +- KubeVirt hosts (`kubevirt-shoryu`/`shushou`/`youko`): avoid mid-flight restarts of the network + stack (`systemd-networkd` / the `br0` bridge) — that can drop the VM network. Use `boot` + serial + reboot for networking or nixpkgs changes. See [hosts/README.md](./hosts/README.md#deploying-kubevirt-hosts). - `just clean`, `just gc`, `just ggc`, and `just game` remove history or amend commits; they are not validation steps and require explicit authorization for their target and scope. diff --git a/hosts/README.md b/hosts/README.md index 4eab9cb9..48983ae5 100644 --- a/hosts/README.md +++ b/hosts/README.md @@ -111,16 +111,15 @@ Use existing hosts as templates. The key files typically include: ## Deploying KubeVirt Hosts -The three KubeVirt hosts (`kubevirt-shoryu`, `kubevirt-shushou`, `kubevirt-youko`) carry the VM -network on the `ovsbr1` Open vSwitch bridge via `ovs-cni`. +The three KubeVirt hosts (`kubevirt-shoryu`, `kubevirt-shushou`, `kubevirt-youko`) put their VMs' +secondary network on the Linux bridge `br0` (attached by the `bridge` CNI plugin); the pod network +is Cilium (flannel disabled). -- `switch` is fine for changes that don't touch the OVS/network stack (e.g. journald or other - service settings). Deploy serially (`-p 1`) and re-check VM reachability afterwards. -- Use `boot` + a serial reboot when the change can restart the OVS units (`ovs-vswitchd` / - `ovsbr1-netdev`) — nixpkgs/Open vSwitch updates, `networking.vswitches` or `systemd.network` - edits, or any broad rebuild. Restarting them recreates the bridge and drops the VM ports. -- If a host loses VM networking, re-add the host-side veths (`ovs-vsctl add-port ovsbr1 `) or - reboot it. +- `switch` is fine for changes that don't touch the network stack (e.g. journald or other service + settings). Deploy serially (`-p 1`) and re-check VM reachability afterwards. +- Use `boot` + a serial reboot for anything that can drop the network mid-flight — `systemd.network` + / `br0` changes, nixpkgs updates, or any broad rebuild. +- If a host loses VM networking, check `br0` and the VM taps, or reboot the host. ## Distributed Building diff --git a/hosts/k8s/kubevirt-shoryu/default.nix b/hosts/k8s/kubevirt-shoryu/default.nix index 34959e0c..5b8cfc36 100644 --- a/hosts/k8s/kubevirt-shoryu/default.nix +++ b/hosts/k8s/kubevirt-shoryu/default.nix @@ -41,10 +41,7 @@ let nodeLabels = [ "node-purpose=kubevirt" ]; - # kubevirt works well with k3s's flannel, - # but has issues with cilium(failed to configure vmi network: setup failed, err: pod link (pod6b4853bd4f2) is missing). - # so we should not disable flannel here. - disableFlannel = false; + # flannel is disabled; Cilium provides the pod network. }; in { diff --git a/hosts/k8s/kubevirt-shushou/default.nix b/hosts/k8s/kubevirt-shushou/default.nix index ed35a6c2..b160612b 100644 --- a/hosts/k8s/kubevirt-shushou/default.nix +++ b/hosts/k8s/kubevirt-shushou/default.nix @@ -38,7 +38,7 @@ let nodeLabels = [ "node-purpose=kubevirt" ]; - disableFlannel = false; + # flannel is disabled; Cilium provides the pod network. }; in { diff --git a/hosts/k8s/kubevirt-youko/default.nix b/hosts/k8s/kubevirt-youko/default.nix index 61baf3ca..6911116c 100644 --- a/hosts/k8s/kubevirt-youko/default.nix +++ b/hosts/k8s/kubevirt-youko/default.nix @@ -38,7 +38,7 @@ let nodeLabels = [ "node-purpose=kubevirt" ]; - disableFlannel = false; + # flannel is disabled; Cilium provides the pod network. }; in { diff --git a/lib/genKubeVirtHostModule.nix b/lib/genKubeVirtHostModule.nix index 95442b4c..ffc85a06 100644 --- a/lib/genKubeVirtHostModule.nix +++ b/lib/genKubeVirtHostModule.nix @@ -59,10 +59,6 @@ in # virt-host-validate qemu libvirt kubevirt # virtctl - - # used by kubernetes' ovs-cni plugin - # https://github.com/k8snetworkplumbingwg/multus-cni - multus-cni ]; # Workaround for longhorn running on NixOS @@ -86,28 +82,22 @@ in networking.useNetworkd = true; systemd.network.enable = true; - # Enable the Open vSwitch as a systemd service - # It's required by kubernetes' ovs-cni plugin. - virtualisation.vswitch = { - enable = true; - # reset the Open vSwitch configuration database to a default configuration on every start of the systemd ovsdb.service - resetOnStart = false; - }; - networking.vswitches = { - # https://github.com/k8snetworkplumbingwg/ovs-cni/blob/main/docs/demo.md - ovsbr1 = { - # Attach the interfaces to OVS bridge - # This interface should not used by the host itself! - interfaces.${iface} = { }; + # Linux bridge for the VMs' secondary network. It replaces the previous OVS + # bridge: the `bridge` CNI plugin attaches the VM taps to br0, and the host + # keeps its own address on br0 (the physical NIC is just a bridge port). + systemd.network.netdevs."10-br0" = { + netdevConfig = { + Name = "br0"; + Kind = "bridge"; }; }; # systemd.services."systemd-networkd".environment.SYSTEMD_LOG_LEVEL = "debug"; - # Set the host's address on the OVS bridge interface instead of the physical interface! + # Set the host's address on the bridge interface instead of the physical interface! systemd.network.networks = { - "10-ovsbr1" = { - matchConfig.Name = [ "ovsbr1" ]; + "10-br0" = { + matchConfig.Name = [ "br0" ]; networkConfig = { Address = [ ipv4WithMask ]; # DNS = nameservers; @@ -131,9 +121,11 @@ in }; "20-${iface}" = { matchConfig.Name = [ iface ]; - networkConfig.LinkLocalAddressing = "no"; - # tell networkd ignore this interface. - # it's managed by openvswitch + networkConfig = { + LinkLocalAddressing = "no"; + # attach the physical NIC to the Linux bridge br0 + Bridge = "br0"; + }; linkConfig.RequiredForOnline = "no"; }; };