Merge pull request #312 from ryan4yin/fix/k3s-test-1-guest-dns

fix(k3s): stabilize guest networking and control-plane metrics

AGENTS.md

@@ -56,22 +56,23 @@ deployments. Keep repository guidance here; reusable global rules live in `agent
 
 ## Command Hazards
 
-- `just up`, `just upp`, and `just up-nix` use `--commit-lock-file`. When a commit is not
-  authorized, use `nix flake update <input>` for a scoped input update without committing.
-- Deployment and upload recipes change systems; use eval/build commands for validation. Remote
-  deployment requires an explicit request. When deployment behavior changes, report the exact `just`
-  command to run.
-- VM hosts (`shoryu`/`shushou`/`youko`): avoid mid-flight restarts of the network
-  stack (`systemd-networkd` / the `br0` bridge) — that can drop the VM network. Use `boot` + serial
-  reboot for networking or nixpkgs changes. See
+- `just eval-host <host>`, `just build-host <host>`, `just build-microvm <guest>`, and `just test`
+  evaluate or build without activating a system. Use these commands for previews and validation.
+- `just up`, `just upp`, and `just up-nix` update flake inputs and commit the lock file. Use
+  `nix flake update <input>` when the update should remain uncommitted.
+- `just shoryu`, `just shushou`, `just youko`, `just lab`, and `just k3s-test` activate systems
+  through Colmena. Use the narrower recipe that matches the intended host scope.
+- `just microvm-deploy <guest> <physical-host> <guest-ip>` installs and activates one MicroVM
+  guest. Deploy guests serially and check the guest Node and host services after each activation.
+- VM hosts (`shoryu`, `shushou`, `youko`) carry the `br0` bridge for their guests. Use the
+  `boot`-based host deployment procedure for network stack or broad nixpkgs changes; see
   [hosts/README.md](./hosts/README.md#deploying-vm-hosts).
-- `just clean`, `just gc`, `just ggc`, and `just game` remove history or amend commits; they are not
-  validation steps and require explicit authorization for their target and scope.
-- Do not use `just penvof` for process inspection: it can expose secret values.
-- MicroVM guests (`k3s-test-1-master-*`) keep their state in
-  `/var/lib/microvms/<name>/{etc,var,home}.img` on their host. Deleting or recreating an image loses
-  the node's identity and cluster state; to resize one, stop the guest and grow the image in place
-  (`truncate` + `e2fsck -f` + `resize2fs`) rather than re-creating it.
+- MicroVM state is stored in `/var/lib/microvms/<name>/{etc,var,home}.img`. Preserve these images
+  when changing the guest configuration.
+- `just clean`, `just gc`, `just ggc`, and `just game` remove state or rewrite history. Use them only
+  for the intended cleanup or history operation.
+- `just penvof` reads a process environment and can expose secrets. Use normal process inspection
+  commands when environment values are not required.
 
 ## Further Context
 

Justfile

@@ -23,6 +23,21 @@ default:
 test:
   nix eval .#evalTests --show-trace --print-build-logs --verbose
 
+# Evaluate a NixOS host configuration without building it.
+[group('nix')]
+eval-host host:
+  nix eval $".#nixosConfigurations.{{host}}.config.system.build.toplevel.drvPath" --raw --show-trace
+
+# Build a NixOS host system closure without changing the current system.
+[group('nix')]
+build-host host:
+  nix build $".#nixosConfigurations.{{host}}.config.system.build.toplevel" --no-link --print-build-logs
+
+# Build a MicroVM runner locally. Deploy it with microvm-deploy so the closure is copied remotely.
+[group('nix')]
+build-microvm guest:
+  nix build $".#nixosConfigurations.{{guest}}.config.microvm.declaredRunner" --no-link --print-build-logs
+
 # Update all the flake inputs
 [group('nix')]
 up:
@@ -183,6 +198,14 @@ reset-launchpad:
 col tag mode="switch":
   colmena apply {{mode}} --on '@{{tag}}' --verbose --show-trace
 
+# Deploy one microVM guest through microvm.nix's SSH deployment workflow.
+# The host is the physical machine running the guest; the guest address is used for activation.
+[linux]
+[group('homelab')]
+microvm-deploy guest host guest_ip:
+  nix run $".#nixosConfigurations.{{guest}}.config.microvm.deploy.installOnHost" -- root@{{host}}
+  nix run $".#nixosConfigurations.{{guest}}.config.microvm.deploy.sshSwitch" -- root@{{guest_ip}} --use-remote-sudo
+
 # Deploy all the VM hosts (physical machines running the VMs)
 [linux]
 [group('homelab')]

hosts/README.md

@@ -125,6 +125,19 @@ runs as microVMs on top, using Cilium as its pod network (flannel disabled).
   / `br0` changes, nixpkgs updates, or any broad rebuild.
 - If a host loses VM networking, check `br0` and the VM taps, or reboot the host.
 
+## Common Build Commands
+
+Use the Just recipes for host configuration checks and builds:
+
+```sh
+just eval-host <host>
+just build-host <host>
+```
+
+`eval-host` only evaluates the configuration. `build-host` builds the system closure without
+activating it. Host activation remains an explicit Colmena operation such as `just shoryu`,
+`just shushou`, or `just youko`.
+
 ## Distributed Building
 
 I usually run the build command on `Ai` and nix will distribute the build to other NixOS machines,

hosts/k8s/README.md

@@ -26,6 +26,34 @@ image has to be built or uploaded; only the volumes above persist. The guest's t
 bridged onto `br0`, and its name is derived from the guest IP (e.g. `192.168.5.114` -> `vm114`, as
 `IFNAMSIZ` caps interface names at 15 characters).
 
+### Updating a running guest
+
+Deploy a guest with the repository's `just` recipe. It evaluates the guest locally, transfers the
+complete Nix closure to the physical host, installs it under `/var/lib/microvms/<name>`, and
+activates the guest:
+
+```sh
+just microvm-deploy <guest> <physical-host> <guest-ip>
+```
+
+For example, `just microvm-deploy k3s-test-1-master-3 shushou 192.168.5.116`.
+The recipe installs the runner on the physical host first, then activates the guest with
+the microvm.nix SSH deployment interface.
+
+Use `just build-microvm <guest>` to build a runner without deploying it. Do not replace the guest
+recipe with a manual symlink operation.
+
+Do not point `/var/lib/microvms/<name>/current` at a store path built on another machine. Nix store
+paths are local until their closure is copied to the target host, and a missing runner makes the
+systemd unit skip startup. Do not delete or recreate `etc.img`, `var.img`, or `home.img`; these
+images hold the guest identity and K3s state.
+
+For a cluster rollout, deploy one guest at a time. Before each restart, record the active runner
+and verify that the new runner exists on the physical host. After each restart, wait for the guest
+Node to become `Ready`, then check the API VIP `/readyz`, Cilium, storage CSI, and any workload
+health gates. If a gate fails, restore the recorded runner through the same host access path and
+start the guest service before proceeding to the next node.
+
 To move an existing VM over in place, copy the host key and `machine-id` from its disk into
 `etc.img`, and `/var/lib/rancher/k3s/server` into `var.img`, **before the first boot** — then it
 keeps its identity and etcd membership instead of bootstrapping a new cluster.

hosts/k8s/k3s-test-1-master-1/default.nix

@@ -25,6 +25,7 @@ let
     # use my own domain & kube-vip's virtual IP for the API server
     # so that the API server can always be accessed even if some nodes are down
     masterHost = "test-cluster-1.writefor.fun";
+    metricsBindAddress = myvars.networking.hostsAddr.${hostName}.ipv4;
     # the control plane only runs the control plane; workloads run on the
     # k3s-test-1-worker-* nodes (labelled node-role.kubernetes.io/worker=true)
     nodeTaints = [ "node-role.kubernetes.io/control-plane:NoSchedule" ];

hosts/k8s/k3s-test-1-master-2/default.nix

@@ -23,6 +23,7 @@ let
     # use my own domain & kube-vip's virtual IP for the API server
     # so that the API server can always be accessed even if some nodes are down
     masterHost = "test-cluster-1.writefor.fun";
+    metricsBindAddress = myvars.networking.hostsAddr.${hostName}.ipv4;
     # the control plane only runs the control plane; workloads run on the
     # k3s-test-1-worker-* nodes (labelled node-role.kubernetes.io/worker=true)
     nodeTaints = [ "node-role.kubernetes.io/control-plane:NoSchedule" ];

hosts/k8s/k3s-test-1-master-3/default.nix

@@ -23,6 +23,7 @@ let
     # use my own domain & kube-vip's virtual IP for the API server
     # so that the API server can always be accessed even if some nodes are down
     masterHost = "test-cluster-1.writefor.fun";
+    metricsBindAddress = myvars.networking.hostsAddr.${hostName}.ipv4;
     # the control plane only runs the control plane; workloads run on the
     # k3s-test-1-worker-* nodes (labelled node-role.kubernetes.io/worker=true)
     nodeTaints = [ "node-role.kubernetes.io/control-plane:NoSchedule" ];

lib/genK3sServerModule.nix

@@ -12,6 +12,8 @@
   clusterInit ? false,
   kubeletExtraArgs ? [ ],
   k3sExtraArgs ? [ ],
+  # Bind embedded K3s control-plane HTTPS metrics to the node address so vmagent can scrape them.
+  metricsBindAddress ? null,
   nodeLabels ? [ ],
   nodeTaints ? [ ],
   disableFlannel ? true,
@@ -76,6 +78,10 @@ in
         ++ (map (label: "--node-label=${label}") nodeLabels)
         ++ (map (taint: "--node-taint=${taint}") nodeTaints)
         ++ (map (arg: "--kubelet-arg=${arg}") kubeletExtraArgs)
+        ++ (lib.optionals (metricsBindAddress != null) [
+          "--kube-controller-manager-arg=bind-address=${metricsBindAddress}"
+          "--kube-scheduler-arg=bind-address=${metricsBindAddress}"
+        ])
         ++ (lib.optionals disableFlannel [ "--flannel-backend=none" ])
         ++ k3sExtraArgs;
       in

lib/genMicrovmGuestModule.nix

@@ -101,6 +101,10 @@ in
       IPv6PrivacyExtensions = false;
       LinkLocalAddressing = "ipv6";
     };
+    # Keep the static resolver below as the only DNS server; router advertisements and DHCPv6
+    # still provide addresses and routes but must not add their own resolver entries.
+    dhcpV6Config.UseDNS = false;
+    ipv6AcceptRAConfig.UseDNS = false;
     routes = [
       {
         Destination = "0.0.0.0/0";