From 069e081fecc6425118aea7d02f7d48b4f30bfd30 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?ryan4yin=20=7C=20=E4=BA=8C=E8=8A=B1?= Date: Thu, 24 Sep 2026 11:13:59 +0800 Subject: [PATCH] fix(k3s): bind control-plane metrics to node addresses (#311) * fix(k3s): bind control-plane metrics to node addresses * docs(microvm): document remote deployment workflow * fix(microvm): deploy runner before guest switch * docs(microvm): standardize on just deployment commands * docs(k3s): explain control-plane metrics address * docs(hosts): move generic build commands to host guide * docs: simplify command operation guidance --- AGENTS.md | 31 ++++++++++++----------- Justfile | 23 +++++++++++++++++ hosts/README.md | 13 ++++++++++ hosts/k8s/README.md | 28 ++++++++++++++++++++ hosts/k8s/k3s-test-1-master-1/default.nix | 1 + hosts/k8s/k3s-test-1-master-2/default.nix | 1 + hosts/k8s/k3s-test-1-master-3/default.nix | 1 + lib/genK3sServerModule.nix | 6 +++++ 8 files changed, 89 insertions(+), 15 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 6440096c..0400b41e 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -56,22 +56,23 @@ deployments. Keep repository guidance here; reusable global rules live in `agent ## Command Hazards -- `just up`, `just upp`, and `just up-nix` use `--commit-lock-file`. When a commit is not - authorized, use `nix flake update ` for a scoped input update without committing. -- Deployment and upload recipes change systems; use eval/build commands for validation. Remote - deployment requires an explicit request. When deployment behavior changes, report the exact `just` - command to run. -- VM hosts (`shoryu`/`shushou`/`youko`): avoid mid-flight restarts of the network - stack (`systemd-networkd` / the `br0` bridge) — that can drop the VM network. Use `boot` + serial - reboot for networking or nixpkgs changes. See +- `just eval-host `, `just build-host `, `just build-microvm `, and `just test` + evaluate or build without activating a system. Use these commands for previews and validation. +- `just up`, `just upp`, and `just up-nix` update flake inputs and commit the lock file. Use + `nix flake update ` when the update should remain uncommitted. +- `just shoryu`, `just shushou`, `just youko`, `just lab`, and `just k3s-test` activate systems + through Colmena. Use the narrower recipe that matches the intended host scope. +- `just microvm-deploy ` installs and activates one MicroVM + guest. Deploy guests serially and check the guest Node and host services after each activation. +- VM hosts (`shoryu`, `shushou`, `youko`) carry the `br0` bridge for their guests. Use the + `boot`-based host deployment procedure for network stack or broad nixpkgs changes; see [hosts/README.md](./hosts/README.md#deploying-vm-hosts). -- `just clean`, `just gc`, `just ggc`, and `just game` remove history or amend commits; they are not - validation steps and require explicit authorization for their target and scope. -- Do not use `just penvof` for process inspection: it can expose secret values. -- MicroVM guests (`k3s-test-1-master-*`) keep their state in - `/var/lib/microvms//{etc,var,home}.img` on their host. Deleting or recreating an image loses - the node's identity and cluster state; to resize one, stop the guest and grow the image in place - (`truncate` + `e2fsck -f` + `resize2fs`) rather than re-creating it. +- MicroVM state is stored in `/var/lib/microvms//{etc,var,home}.img`. Preserve these images + when changing the guest configuration. +- `just clean`, `just gc`, `just ggc`, and `just game` remove state or rewrite history. Use them only + for the intended cleanup or history operation. +- `just penvof` reads a process environment and can expose secrets. Use normal process inspection + commands when environment values are not required. ## Further Context diff --git a/Justfile b/Justfile index a5e9977c..138acd6f 100644 --- a/Justfile +++ b/Justfile @@ -23,6 +23,21 @@ default: test: nix eval .#evalTests --show-trace --print-build-logs --verbose +# Evaluate a NixOS host configuration without building it. +[group('nix')] +eval-host host: + nix eval $".#nixosConfigurations.{{host}}.config.system.build.toplevel.drvPath" --raw --show-trace + +# Build a NixOS host system closure without changing the current system. +[group('nix')] +build-host host: + nix build $".#nixosConfigurations.{{host}}.config.system.build.toplevel" --no-link --print-build-logs + +# Build a MicroVM runner locally. Deploy it with microvm-deploy so the closure is copied remotely. +[group('nix')] +build-microvm guest: + nix build $".#nixosConfigurations.{{guest}}.config.microvm.declaredRunner" --no-link --print-build-logs + # Update all the flake inputs [group('nix')] up: @@ -183,6 +198,14 @@ reset-launchpad: col tag mode="switch": colmena apply {{mode}} --on '@{{tag}}' --verbose --show-trace +# Deploy one microVM guest through microvm.nix's SSH deployment workflow. +# The host is the physical machine running the guest; the guest address is used for activation. +[linux] +[group('homelab')] +microvm-deploy guest host guest_ip: + nix run $".#nixosConfigurations.{{guest}}.config.microvm.deploy.installOnHost" -- root@{{host}} + nix run $".#nixosConfigurations.{{guest}}.config.microvm.deploy.sshSwitch" -- root@{{guest_ip}} --use-remote-sudo + # Deploy all the VM hosts (physical machines running the VMs) [linux] [group('homelab')] diff --git a/hosts/README.md b/hosts/README.md index b42b1cbd..edd5088f 100644 --- a/hosts/README.md +++ b/hosts/README.md @@ -125,6 +125,19 @@ runs as microVMs on top, using Cilium as its pod network (flannel disabled). / `br0` changes, nixpkgs updates, or any broad rebuild. - If a host loses VM networking, check `br0` and the VM taps, or reboot the host. +## Common Build Commands + +Use the Just recipes for host configuration checks and builds: + +```sh +just eval-host +just build-host +``` + +`eval-host` only evaluates the configuration. `build-host` builds the system closure without +activating it. Host activation remains an explicit Colmena operation such as `just shoryu`, +`just shushou`, or `just youko`. + ## Distributed Building I usually run the build command on `Ai` and nix will distribute the build to other NixOS machines, diff --git a/hosts/k8s/README.md b/hosts/k8s/README.md index 852a2b1f..f4ac59e3 100644 --- a/hosts/k8s/README.md +++ b/hosts/k8s/README.md @@ -26,6 +26,34 @@ image has to be built or uploaded; only the volumes above persist. The guest's t bridged onto `br0`, and its name is derived from the guest IP (e.g. `192.168.5.114` -> `vm114`, as `IFNAMSIZ` caps interface names at 15 characters). +### Updating a running guest + +Deploy a guest with the repository's `just` recipe. It evaluates the guest locally, transfers the +complete Nix closure to the physical host, installs it under `/var/lib/microvms/`, and +activates the guest: + +```sh +just microvm-deploy +``` + +For example, `just microvm-deploy k3s-test-1-master-3 shushou 192.168.5.116`. +The recipe installs the runner on the physical host first, then activates the guest with +the microvm.nix SSH deployment interface. + +Use `just build-microvm ` to build a runner without deploying it. Do not replace the guest +recipe with a manual symlink operation. + +Do not point `/var/lib/microvms//current` at a store path built on another machine. Nix store +paths are local until their closure is copied to the target host, and a missing runner makes the +systemd unit skip startup. Do not delete or recreate `etc.img`, `var.img`, or `home.img`; these +images hold the guest identity and K3s state. + +For a cluster rollout, deploy one guest at a time. Before each restart, record the active runner +and verify that the new runner exists on the physical host. After each restart, wait for the guest +Node to become `Ready`, then check the API VIP `/readyz`, Cilium, storage CSI, and any workload +health gates. If a gate fails, restore the recorded runner through the same host access path and +start the guest service before proceeding to the next node. + To move an existing VM over in place, copy the host key and `machine-id` from its disk into `etc.img`, and `/var/lib/rancher/k3s/server` into `var.img`, **before the first boot** — then it keeps its identity and etcd membership instead of bootstrapping a new cluster. diff --git a/hosts/k8s/k3s-test-1-master-1/default.nix b/hosts/k8s/k3s-test-1-master-1/default.nix index 6a42651f..1266ded6 100644 --- a/hosts/k8s/k3s-test-1-master-1/default.nix +++ b/hosts/k8s/k3s-test-1-master-1/default.nix @@ -25,6 +25,7 @@ let # use my own domain & kube-vip's virtual IP for the API server # so that the API server can always be accessed even if some nodes are down masterHost = "test-cluster-1.writefor.fun"; + metricsBindAddress = myvars.networking.hostsAddr.${hostName}.ipv4; # the control plane only runs the control plane; workloads run on the # k3s-test-1-worker-* nodes (labelled node-role.kubernetes.io/worker=true) nodeTaints = [ "node-role.kubernetes.io/control-plane:NoSchedule" ]; diff --git a/hosts/k8s/k3s-test-1-master-2/default.nix b/hosts/k8s/k3s-test-1-master-2/default.nix index 7c5f1200..cea65004 100644 --- a/hosts/k8s/k3s-test-1-master-2/default.nix +++ b/hosts/k8s/k3s-test-1-master-2/default.nix @@ -23,6 +23,7 @@ let # use my own domain & kube-vip's virtual IP for the API server # so that the API server can always be accessed even if some nodes are down masterHost = "test-cluster-1.writefor.fun"; + metricsBindAddress = myvars.networking.hostsAddr.${hostName}.ipv4; # the control plane only runs the control plane; workloads run on the # k3s-test-1-worker-* nodes (labelled node-role.kubernetes.io/worker=true) nodeTaints = [ "node-role.kubernetes.io/control-plane:NoSchedule" ]; diff --git a/hosts/k8s/k3s-test-1-master-3/default.nix b/hosts/k8s/k3s-test-1-master-3/default.nix index 8d2fd2e0..fb0356d2 100644 --- a/hosts/k8s/k3s-test-1-master-3/default.nix +++ b/hosts/k8s/k3s-test-1-master-3/default.nix @@ -23,6 +23,7 @@ let # use my own domain & kube-vip's virtual IP for the API server # so that the API server can always be accessed even if some nodes are down masterHost = "test-cluster-1.writefor.fun"; + metricsBindAddress = myvars.networking.hostsAddr.${hostName}.ipv4; # the control plane only runs the control plane; workloads run on the # k3s-test-1-worker-* nodes (labelled node-role.kubernetes.io/worker=true) nodeTaints = [ "node-role.kubernetes.io/control-plane:NoSchedule" ]; diff --git a/lib/genK3sServerModule.nix b/lib/genK3sServerModule.nix index d77f175a..5a1fecbe 100644 --- a/lib/genK3sServerModule.nix +++ b/lib/genK3sServerModule.nix @@ -12,6 +12,8 @@ clusterInit ? false, kubeletExtraArgs ? [ ], k3sExtraArgs ? [ ], + # Bind embedded K3s control-plane HTTPS metrics to the node address so vmagent can scrape them. + metricsBindAddress ? null, nodeLabels ? [ ], nodeTaints ? [ ], disableFlannel ? true, @@ -76,6 +78,10 @@ in ++ (map (label: "--node-label=${label}") nodeLabels) ++ (map (taint: "--node-taint=${taint}") nodeTaints) ++ (map (arg: "--kubelet-arg=${arg}") kubeletExtraArgs) + ++ (lib.optionals (metricsBindAddress != null) [ + "--kube-controller-manager-arg=bind-address=${metricsBindAddress}" + "--kube-scheduler-arg=bind-address=${metricsBindAddress}" + ]) ++ (lib.optionals disableFlannel [ "--flannel-backend=none" ]) ++ k3sExtraArgs; in