From 9a6923b4e9d0e502b1a91e334bb7dcd9c5079d65 Mon Sep 17 00:00:00 2001 From: Ryan Yin Date: Thu, 24 Sep 2026 13:35:33 +0800 Subject: [PATCH] fix(k3s): drop Home Manager from test masters Keep the six node configuration intact while removing the failing user activation from the three control-plane MicroVMs. Update the regression test and current K3s configuration notes. --- home/hosts/linux/k3s-test-1-master-1.nix | 3 -- home/hosts/linux/k3s-test-1-master-2.nix | 3 -- home/hosts/linux/k3s-test-1-master-3.nix | 3 -- hosts/k8s/README.md | 35 ++++++------------- hosts/k8s/k3s-test-1-master-1/default.nix | 3 +- hosts/k8s/k3s-test-1-master-2/default.nix | 3 +- hosts/k8s/k3s-test-1-master-3/default.nix | 3 +- lib/genK3sServerModule.nix | 6 ++-- lib/genMicrovmGuestModule.nix | 2 +- .../x86_64-linux/src/k3s-test-1-master-1.nix | 3 -- .../x86_64-linux/src/k3s-test-1-master-2.nix | 3 -- .../x86_64-linux/src/k3s-test-1-master-3.nix | 3 -- .../tests/home-manager/expected.nix | 1 - .../x86_64-linux/tests/home-manager/expr.nix | 1 - .../k3s-master-home-manager/expected.nix | 6 ++++ .../tests/k3s-master-home-manager/expr.nix | 16 +++++++++ 16 files changed, 40 insertions(+), 54 deletions(-) delete mode 100644 home/hosts/linux/k3s-test-1-master-1.nix delete mode 100644 home/hosts/linux/k3s-test-1-master-2.nix delete mode 100644 home/hosts/linux/k3s-test-1-master-3.nix create mode 100644 outputs/x86_64-linux/tests/k3s-master-home-manager/expected.nix create mode 100644 outputs/x86_64-linux/tests/k3s-master-home-manager/expr.nix diff --git a/home/hosts/linux/k3s-test-1-master-1.nix b/home/hosts/linux/k3s-test-1-master-1.nix deleted file mode 100644 index a67103bf..00000000 --- a/home/hosts/linux/k3s-test-1-master-1.nix +++ /dev/null @@ -1,3 +0,0 @@ -{ - imports = [ ../../linux/core.nix ]; -} diff --git a/home/hosts/linux/k3s-test-1-master-2.nix b/home/hosts/linux/k3s-test-1-master-2.nix deleted file mode 100644 index a67103bf..00000000 --- a/home/hosts/linux/k3s-test-1-master-2.nix +++ /dev/null @@ -1,3 +0,0 @@ -{ - imports = [ ../../linux/core.nix ]; -} diff --git a/home/hosts/linux/k3s-test-1-master-3.nix b/home/hosts/linux/k3s-test-1-master-3.nix deleted file mode 100644 index a67103bf..00000000 --- a/home/hosts/linux/k3s-test-1-master-3.nix +++ /dev/null @@ -1,3 +0,0 @@ -{ - imports = [ ../../linux/core.nix ]; -} diff --git a/hosts/k8s/README.md b/hosts/k8s/README.md index f4ac59e3..b7431a12 100644 --- a/hosts/k8s/README.md +++ b/hosts/k8s/README.md @@ -36,27 +36,18 @@ activates the guest: just microvm-deploy ``` -For example, `just microvm-deploy k3s-test-1-master-3 shushou 192.168.5.116`. -The recipe installs the runner on the physical host first, then activates the guest with -the microvm.nix SSH deployment interface. +For example, `just microvm-deploy k3s-test-1-master-3 shushou 192.168.5.116`. The recipe installs +the runner on the physical host first, then activates the guest with the microvm.nix SSH deployment +interface. -Use `just build-microvm ` to build a runner without deploying it. Do not replace the guest -recipe with a manual symlink operation. +Use `just build-microvm ` to build a runner without deploying it. Preserve the guest's +`etc.img`, `var.img`, and `home.img` when updating it; they contain its identity and K3s state. -Do not point `/var/lib/microvms//current` at a store path built on another machine. Nix store -paths are local until their closure is copied to the target host, and a missing runner makes the -systemd unit skip startup. Do not delete or recreate `etc.img`, `var.img`, or `home.img`; these -images hold the guest identity and K3s state. - -For a cluster rollout, deploy one guest at a time. Before each restart, record the active runner -and verify that the new runner exists on the physical host. After each restart, wait for the guest -Node to become `Ready`, then check the API VIP `/readyz`, Cilium, storage CSI, and any workload -health gates. If a gate fails, restore the recorded runner through the same host access path and -start the guest service before proceeding to the next node. - -To move an existing VM over in place, copy the host key and `machine-id` from its disk into -`etc.img`, and `/var/lib/rancher/k3s/server` into `var.img`, **before the first boot** — then it -keeps its identity and etcd membership instead of bootstrapping a new cluster. +For a cluster rollout, deploy one guest at a time. Before each restart, record the active runner and +verify that the new runner exists on the physical host. After each restart, wait for the guest Node +to become `Ready`, then check the API VIP `/readyz`, Cilium, storage CSI, and any workload health +gates. If a gate fails, restore the recorded runner through the same host access path and start the +guest service before proceeding to the next node. 1. `k3s-test-1-master-{1,2,3}` — control plane, running as microVMs; tainted `node-role.kubernetes.io/control-plane:NoSchedule` @@ -74,12 +65,6 @@ from scratch gets it automatically. The workers carry no `node-role.kubernetes.i kubelet refuses to self-assign the reserved `node-role.kubernetes.io/*` labels via `--node-label`, and placement is enforced by the control-plane taint instead. -## TODO / Known issues - -- **The USB HDD bridge is flaky.** youko's two HDDs sit behind a JMicron JMS567 USB-SATA bridge that - keeps resetting (`dmesg` on `youko`). Find out how often it resets and how much it matters before - putting anything critical (e.g. an NFS export) on it. - ## Kubernetes Resources Kubernetes resources are deployed and managed separately through diff --git a/hosts/k8s/k3s-test-1-master-1/default.nix b/hosts/k8s/k3s-test-1-master-1/default.nix index 1266ded6..38c31f94 100644 --- a/hosts/k8s/k3s-test-1-master-1/default.nix +++ b/hosts/k8s/k3s-test-1-master-1/default.nix @@ -26,8 +26,7 @@ let # so that the API server can always be accessed even if some nodes are down masterHost = "test-cluster-1.writefor.fun"; metricsBindAddress = myvars.networking.hostsAddr.${hostName}.ipv4; - # the control plane only runs the control plane; workloads run on the - # k3s-test-1-worker-* nodes (labelled node-role.kubernetes.io/worker=true) + # Workloads run on the worker nodes; they do not use a reserved worker-role label. nodeTaints = [ "node-role.kubernetes.io/control-plane:NoSchedule" ]; # k3sExtraArgs = [ diff --git a/hosts/k8s/k3s-test-1-master-2/default.nix b/hosts/k8s/k3s-test-1-master-2/default.nix index cea65004..28525d0f 100644 --- a/hosts/k8s/k3s-test-1-master-2/default.nix +++ b/hosts/k8s/k3s-test-1-master-2/default.nix @@ -24,8 +24,7 @@ let # so that the API server can always be accessed even if some nodes are down masterHost = "test-cluster-1.writefor.fun"; metricsBindAddress = myvars.networking.hostsAddr.${hostName}.ipv4; - # the control plane only runs the control plane; workloads run on the - # k3s-test-1-worker-* nodes (labelled node-role.kubernetes.io/worker=true) + # Workloads run on the worker nodes; they do not use a reserved worker-role label. nodeTaints = [ "node-role.kubernetes.io/control-plane:NoSchedule" ]; }; in diff --git a/hosts/k8s/k3s-test-1-master-3/default.nix b/hosts/k8s/k3s-test-1-master-3/default.nix index fb0356d2..4b6d3642 100644 --- a/hosts/k8s/k3s-test-1-master-3/default.nix +++ b/hosts/k8s/k3s-test-1-master-3/default.nix @@ -24,8 +24,7 @@ let # so that the API server can always be accessed even if some nodes are down masterHost = "test-cluster-1.writefor.fun"; metricsBindAddress = myvars.networking.hostsAddr.${hostName}.ipv4; - # the control plane only runs the control plane; workloads run on the - # k3s-test-1-worker-* nodes (labelled node-role.kubernetes.io/worker=true) + # Workloads run on the worker nodes; they do not use a reserved worker-role label. nodeTaints = [ "node-role.kubernetes.io/control-plane:NoSchedule" ]; }; in diff --git a/lib/genK3sServerModule.nix b/lib/genK3sServerModule.nix index 5a1fecbe..f68f0712 100644 --- a/lib/genK3sServerModule.nix +++ b/lib/genK3sServerModule.nix @@ -63,8 +63,10 @@ in flagList = [ "--write-kubeconfig=${kubeconfigFile}" "--write-kubeconfig-mode=600" + # The Istio ingress Service uses NodePorts 80 and 443. "--service-node-port-range=80-32767" - "--kube-apiserver-arg='--allow-privileged=true'" # required by the former KubeVirt VMs + # The NFS CSI node plugin runs a privileged container to mount volumes. + "--kube-apiserver-arg='--allow-privileged=true'" "--data-dir /var/lib/rancher/k3s" "--etcd-expose-metrics=true" "--etcd-snapshot-schedule-cron='0 */12 * * *'" @@ -72,7 +74,7 @@ in "--disable-helm-controller" # we use fluxcd instead "--disable=traefik" # deploy our own ingress controller instead "--disable=servicelb" # we use kube-vip instead - "--disable-network-policy" + "--disable-network-policy" # Cilium enforces network policies instead "--tls-san=${masterHost}" ] ++ (map (label: "--node-label=${label}") nodeLabels) diff --git a/lib/genMicrovmGuestModule.nix b/lib/genMicrovmGuestModule.nix index 170b03ff..ac07bb98 100644 --- a/lib/genMicrovmGuestModule.nix +++ b/lib/genMicrovmGuestModule.nix @@ -2,7 +2,7 @@ pkgs, hostName, networking, - # vCPU count and RAM (MiB) -- mirrors the former KubeVirt instancetype + # vCPU count and RAM (MiB) for the QEMU guest vcpu, mem, # persistent volume sizes in MiB diff --git a/outputs/x86_64-linux/src/k3s-test-1-master-1.nix b/outputs/x86_64-linux/src/k3s-test-1-master-1.nix index 1ec23a3b..00b105ee 100644 --- a/outputs/x86_64-linux/src/k3s-test-1-master-1.nix +++ b/outputs/x86_64-linux/src/k3s-test-1-master-1.nix @@ -26,9 +26,6 @@ let inputs.microvm.nixosModules.microvm { modules.secrets.server.kubernetes.enable = true; } ]; - home-modules = map mylib.relativeToRoot [ - "home/hosts/linux/${name}.nix" - ]; }; systemArgs = modules // args; diff --git a/outputs/x86_64-linux/src/k3s-test-1-master-2.nix b/outputs/x86_64-linux/src/k3s-test-1-master-2.nix index 062aed88..3bc8f477 100644 --- a/outputs/x86_64-linux/src/k3s-test-1-master-2.nix +++ b/outputs/x86_64-linux/src/k3s-test-1-master-2.nix @@ -26,9 +26,6 @@ let inputs.microvm.nixosModules.microvm { modules.secrets.server.kubernetes.enable = true; } ]; - home-modules = map mylib.relativeToRoot [ - "home/hosts/linux/${name}.nix" - ]; }; systemArgs = modules // args; diff --git a/outputs/x86_64-linux/src/k3s-test-1-master-3.nix b/outputs/x86_64-linux/src/k3s-test-1-master-3.nix index e0ad4c80..c0d6393f 100644 --- a/outputs/x86_64-linux/src/k3s-test-1-master-3.nix +++ b/outputs/x86_64-linux/src/k3s-test-1-master-3.nix @@ -26,9 +26,6 @@ let inputs.microvm.nixosModules.microvm { modules.secrets.server.kubernetes.enable = true; } ]; - home-modules = map mylib.relativeToRoot [ - "home/hosts/linux/${name}.nix" - ]; }; systemArgs = modules // args; diff --git a/outputs/x86_64-linux/tests/home-manager/expected.nix b/outputs/x86_64-linux/tests/home-manager/expected.nix index 7c3cb241..0b54552e 100644 --- a/outputs/x86_64-linux/tests/home-manager/expected.nix +++ b/outputs/x86_64-linux/tests/home-manager/expected.nix @@ -7,7 +7,6 @@ let hosts = [ "ai-niri" "ruby" - "k3s-test-1-master-1" ]; in lib.genAttrs hosts (_: "/home/${username}") diff --git a/outputs/x86_64-linux/tests/home-manager/expr.nix b/outputs/x86_64-linux/tests/home-manager/expr.nix index 517a3188..7f41a640 100644 --- a/outputs/x86_64-linux/tests/home-manager/expr.nix +++ b/outputs/x86_64-linux/tests/home-manager/expr.nix @@ -8,7 +8,6 @@ let hosts = [ "ai-niri" "ruby" - "k3s-test-1-master-1" ]; in lib.genAttrs hosts ( diff --git a/outputs/x86_64-linux/tests/k3s-master-home-manager/expected.nix b/outputs/x86_64-linux/tests/k3s-master-home-manager/expected.nix new file mode 100644 index 00000000..4263ce54 --- /dev/null +++ b/outputs/x86_64-linux/tests/k3s-master-home-manager/expected.nix @@ -0,0 +1,6 @@ +{ lib, ... }: +lib.genAttrs [ + "k3s-test-1-master-1" + "k3s-test-1-master-2" + "k3s-test-1-master-3" +] (_: false) diff --git a/outputs/x86_64-linux/tests/k3s-master-home-manager/expr.nix b/outputs/x86_64-linux/tests/k3s-master-home-manager/expr.nix new file mode 100644 index 00000000..d3b9c593 --- /dev/null +++ b/outputs/x86_64-linux/tests/k3s-master-home-manager/expr.nix @@ -0,0 +1,16 @@ +{ + lib, + myvars, + outputs, +}: +lib.genAttrs + [ + "k3s-test-1-master-1" + "k3s-test-1-master-2" + "k3s-test-1-master-3" + ] + ( + name: + builtins.hasAttr "home-manager-${myvars.username}" + outputs.nixosConfigurations.${name}.config.systemd.services + )