From 64cad91401e30a45d9bb0d20c47f9bd92813ce60 Mon Sep 17 00:00:00 2001 From: Ryan Yin Date: Wed, 23 Sep 2026 10:57:25 +0800 Subject: [PATCH] fix(k8s): unblock the k3s-test-1 workers kubelet refuses to self-assign the reserved node-role.kubernetes.io/* labels, so the worker --node-label made the agent crash-loop. Drop it and rely on the masters' control-plane taint for placement; document that the taint only lands via registerWithTaints on first registration, so it was applied by hand here. --- hosts/k8s/README.md | 9 ++++++++- hosts/k8s/k3s-test-1-worker-1/default.nix | 4 ++-- hosts/k8s/k3s-test-1-worker-2/default.nix | 4 ++-- hosts/k8s/k3s-test-1-worker-3/default.nix | 4 ++-- 4 files changed, 14 insertions(+), 7 deletions(-) diff --git a/hosts/k8s/README.md b/hosts/k8s/README.md index e4232741..852a2b1f 100644 --- a/hosts/k8s/README.md +++ b/hosts/k8s/README.md @@ -33,12 +33,19 @@ keeps its identity and etcd membership instead of bootstrapping a new cluster. 1. `k3s-test-1-master-{1,2,3}` — control plane, running as microVMs; tainted `node-role.kubernetes.io/control-plane:NoSchedule` 1. `k3s-test-1-worker-{1,2,3}` — workloads, running as microVMs - (`node-role.kubernetes.io/worker=true`) Placement: `worker-1` (4 vCPU / 16 GiB) runs on `shoryu`; `worker-2` (4 vCPU / 16 GiB) and `worker-3` (2 vCPU / 8 GiB) run on `shushou`. `youko` has no worker because it has the least free memory and also runs the homelab services. +The master taint comes from kubelet's `registerWithTaints`, which only takes effect when the Node +object is first created. These masters are long-lived and predate the config, so the taint was +applied once by hand +(`kubectl taint nodes node-role.kubernetes.io/control-plane=:NoSchedule`); a node rebuilt +from scratch gets it automatically. The workers carry no `node-role.kubernetes.io/worker` label: +kubelet refuses to self-assign the reserved `node-role.kubernetes.io/*` labels via `--node-label`, +and placement is enforced by the control-plane taint instead. + ## TODO / Known issues - **The USB HDD bridge is flaky.** youko's two HDDs sit behind a JMicron JMS567 USB-SATA bridge that diff --git a/hosts/k8s/k3s-test-1-worker-1/default.nix b/hosts/k8s/k3s-test-1-worker-1/default.nix index 0edca784..e67b39b8 100644 --- a/hosts/k8s/k3s-test-1-worker-1/default.nix +++ b/hosts/k8s/k3s-test-1-worker-1/default.nix @@ -22,8 +22,8 @@ let # use my own domain & kube-vip's virtual IP for the API server # so that the API server can always be accessed even if some nodes are down masterHost = "test-cluster-1.writefor.fun"; - # workloads run on the workers; the masters are tainted (see k3s-test-1-master-*) - nodeLabels = [ "node-role.kubernetes.io/worker=true" ]; + # Placement is enforced by the masters' control-plane taint. No node label: + # kubelet refuses to self-assign the reserved node-role.kubernetes.io/* labels. }; in { diff --git a/hosts/k8s/k3s-test-1-worker-2/default.nix b/hosts/k8s/k3s-test-1-worker-2/default.nix index a6cf3986..bbf54936 100644 --- a/hosts/k8s/k3s-test-1-worker-2/default.nix +++ b/hosts/k8s/k3s-test-1-worker-2/default.nix @@ -22,8 +22,8 @@ let # use my own domain & kube-vip's virtual IP for the API server # so that the API server can always be accessed even if some nodes are down masterHost = "test-cluster-1.writefor.fun"; - # workloads run on the workers; the masters are tainted (see k3s-test-1-master-*) - nodeLabels = [ "node-role.kubernetes.io/worker=true" ]; + # Placement is enforced by the masters' control-plane taint. No node label: + # kubelet refuses to self-assign the reserved node-role.kubernetes.io/* labels. }; in { diff --git a/hosts/k8s/k3s-test-1-worker-3/default.nix b/hosts/k8s/k3s-test-1-worker-3/default.nix index 95de1903..2e9d3029 100644 --- a/hosts/k8s/k3s-test-1-worker-3/default.nix +++ b/hosts/k8s/k3s-test-1-worker-3/default.nix @@ -22,8 +22,8 @@ let # use my own domain & kube-vip's virtual IP for the API server # so that the API server can always be accessed even if some nodes are down masterHost = "test-cluster-1.writefor.fun"; - # workloads run on the workers; the masters are tainted (see k3s-test-1-master-*) - nodeLabels = [ "node-role.kubernetes.io/worker=true" ]; + # Placement is enforced by the masters' control-plane taint. No node label: + # kubelet refuses to self-assign the reserved node-role.kubernetes.io/* labels. }; in {