Merge pull request #313 from ryan4yin/fix/k3s-test-1-master-config-cleanup

fix(k3s): drop Home Manager from test masters
2026-11-13 12:13:36 +01:00 · 2026-09-24 14:12:04 +08:00
commit c3a8d99ceb
@@ -1,3 +0,0 @@
-{
-  imports = [ ../../linux/core.nix ];
-}
@@ -1,3 +0,0 @@
-{
-  imports = [ ../../linux/core.nix ];
-}
@@ -1,3 +0,0 @@
-{
-  imports = [ ../../linux/core.nix ];
-}
@@ -36,27 +36,18 @@ activates the guest:
 just microvm-deploy <guest> <physical-host> <guest-ip>
 ```

-For example, `just microvm-deploy k3s-test-1-master-3 shushou 192.168.5.116`.
-The recipe installs the runner on the physical host first, then activates the guest with
-the microvm.nix SSH deployment interface.
+For example, `just microvm-deploy k3s-test-1-master-3 shushou 192.168.5.116`. The recipe installs
+the runner on the physical host first, then activates the guest with the microvm.nix SSH deployment
+interface.

-Use `just build-microvm <guest>` to build a runner without deploying it. Do not replace the guest
-recipe with a manual symlink operation.
+Use `just build-microvm <guest>` to build a runner without deploying it. Preserve the guest's
+`etc.img`, `var.img`, and `home.img` when updating it; they contain its identity and K3s state.

-Do not point `/var/lib/microvms/<name>/current` at a store path built on another machine. Nix store
-paths are local until their closure is copied to the target host, and a missing runner makes the
-systemd unit skip startup. Do not delete or recreate `etc.img`, `var.img`, or `home.img`; these
-images hold the guest identity and K3s state.
-
-For a cluster rollout, deploy one guest at a time. Before each restart, record the active runner
-and verify that the new runner exists on the physical host. After each restart, wait for the guest
-Node to become `Ready`, then check the API VIP `/readyz`, Cilium, storage CSI, and any workload
-health gates. If a gate fails, restore the recorded runner through the same host access path and
-start the guest service before proceeding to the next node.
-
-To move an existing VM over in place, copy the host key and `machine-id` from its disk into
-`etc.img`, and `/var/lib/rancher/k3s/server` into `var.img`, **before the first boot** — then it
-keeps its identity and etcd membership instead of bootstrapping a new cluster.
+For a cluster rollout, deploy one guest at a time. Before each restart, record the active runner and
+verify that the new runner exists on the physical host. After each restart, wait for the guest Node
+to become `Ready`, then check the API VIP `/readyz`, Cilium, storage CSI, and any workload health
+gates. If a gate fails, restore the recorded runner through the same host access path and start the
+guest service before proceeding to the next node.

 1. `k3s-test-1-master-{1,2,3}` — control plane, running as microVMs; tainted
   `node-role.kubernetes.io/control-plane:NoSchedule`
@@ -74,12 +65,6 @@ from scratch gets it automatically. The workers carry no `node-role.kubernetes.i
 kubelet refuses to self-assign the reserved `node-role.kubernetes.io/*` labels via `--node-label`,
 and placement is enforced by the control-plane taint instead.

-## TODO / Known issues
-
- **The USB HDD bridge is flaky.** youko's two HDDs sit behind a JMicron JMS567 USB-SATA bridge that
-  keeps resetting (`dmesg` on `youko`). Find out how often it resets and how much it matters before
-  putting anything critical (e.g. an NFS export) on it.
-
 ## Kubernetes Resources

 Kubernetes resources are deployed and managed separately through
@@ -26,8 +26,7 @@ let
    # so that the API server can always be accessed even if some nodes are down
    masterHost = "test-cluster-1.writefor.fun";
    metricsBindAddress = myvars.networking.hostsAddr.${hostName}.ipv4;
-    # the control plane only runs the control plane; workloads run on the
-    # k3s-test-1-worker-* nodes (labelled node-role.kubernetes.io/worker=true)
+    # Workloads run on the worker nodes; they do not use a reserved worker-role label.
    nodeTaints = [ "node-role.kubernetes.io/control-plane:NoSchedule" ];

    # k3sExtraArgs = [
@@ -24,8 +24,7 @@ let
    # so that the API server can always be accessed even if some nodes are down
    masterHost = "test-cluster-1.writefor.fun";
    metricsBindAddress = myvars.networking.hostsAddr.${hostName}.ipv4;
-    # the control plane only runs the control plane; workloads run on the
-    # k3s-test-1-worker-* nodes (labelled node-role.kubernetes.io/worker=true)
+    # Workloads run on the worker nodes; they do not use a reserved worker-role label.
    nodeTaints = [ "node-role.kubernetes.io/control-plane:NoSchedule" ];
  };
 in
@@ -24,8 +24,7 @@ let
    # so that the API server can always be accessed even if some nodes are down
    masterHost = "test-cluster-1.writefor.fun";
    metricsBindAddress = myvars.networking.hostsAddr.${hostName}.ipv4;
-    # the control plane only runs the control plane; workloads run on the
-    # k3s-test-1-worker-* nodes (labelled node-role.kubernetes.io/worker=true)
+    # Workloads run on the worker nodes; they do not use a reserved worker-role label.
    nodeTaints = [ "node-role.kubernetes.io/control-plane:NoSchedule" ];
  };
 in
@@ -63,8 +63,10 @@ in
        flagList = [
          "--write-kubeconfig=${kubeconfigFile}"
          "--write-kubeconfig-mode=600"
+          # The Istio ingress Service uses NodePorts 80 and 443.
          "--service-node-port-range=80-32767"
-          "--kube-apiserver-arg='--allow-privileged=true'" # required by the former KubeVirt VMs
+          # The NFS CSI node plugin runs a privileged container to mount volumes.
+          "--kube-apiserver-arg='--allow-privileged=true'"
          "--data-dir /var/lib/rancher/k3s"
          "--etcd-expose-metrics=true"
          "--etcd-snapshot-schedule-cron='0 */12 * * *'"
@@ -72,7 +74,7 @@ in
          "--disable-helm-controller" # we use fluxcd instead
          "--disable=traefik" # deploy our own ingress controller instead
          "--disable=servicelb" # we use kube-vip instead
-          "--disable-network-policy"
+          "--disable-network-policy" # Cilium enforces network policies instead
          "--tls-san=${masterHost}"
        ]
        ++ (map (label: "--node-label=${label}") nodeLabels)
@@ -2,7 +2,7 @@
  pkgs,
  hostName,
  networking,
-  # vCPU count and RAM (MiB) -- mirrors the former KubeVirt instancetype
+  # vCPU count and RAM (MiB) for the QEMU guest
  vcpu,
  mem,
  # persistent volume sizes in MiB
@@ -26,9 +26,6 @@ let
        inputs.microvm.nixosModules.microvm
        { modules.secrets.server.kubernetes.enable = true; }
      ];
-    home-modules = map mylib.relativeToRoot [
-      "home/hosts/linux/${name}.nix"
-    ];
  };

  systemArgs = modules // args;
@@ -26,9 +26,6 @@ let
        inputs.microvm.nixosModules.microvm
        { modules.secrets.server.kubernetes.enable = true; }
      ];
-    home-modules = map mylib.relativeToRoot [
-      "home/hosts/linux/${name}.nix"
-    ];
  };

  systemArgs = modules // args;
@@ -26,9 +26,6 @@ let
        inputs.microvm.nixosModules.microvm
        { modules.secrets.server.kubernetes.enable = true; }
      ];
-    home-modules = map mylib.relativeToRoot [
-      "home/hosts/linux/${name}.nix"
-    ];
  };

  systemArgs = modules // args;
@@ -7,7 +7,6 @@ let
  hosts = [
    "ai-niri"
    "ruby"
-    "k3s-test-1-master-1"
  ];
 in
 lib.genAttrs hosts (_: "/home/${username}")
@@ -8,7 +8,6 @@ let
  hosts = [
    "ai-niri"
    "ruby"
-    "k3s-test-1-master-1"
  ];
 in
 lib.genAttrs hosts (
@@ -0,0 +1,6 @@
+{ lib, ... }:
+lib.genAttrs [
+  "k3s-test-1-master-1"
+  "k3s-test-1-master-2"
+  "k3s-test-1-master-3"
+] (_: false)
@@ -0,0 +1,16 @@
+{
+  lib,
+  myvars,
+  outputs,
+}:
+lib.genAttrs
+  [
+    "k3s-test-1-master-1"
+    "k3s-test-1-master-2"
+    "k3s-test-1-master-3"
+  ]
+  (
+    name:
+    builtins.hasAttr "home-manager-${myvars.username}"
+      outputs.nixosConfigurations.${name}.config.systemd.services
+  )