From b19818ef44973d483173c98f6e28a84e171bb01e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?ryan4yin=20=7C=20=E4=BA=8C=E8=8A=B1?= Date: Mon, 21 Sep 2026 23:09:13 +0800 Subject: [PATCH] feat(k8s): add the k3s-test-1 worker nodes (#307) Add worker-1 (4 vCPU / 16 GiB) on shoryu, and worker-2 (4 vCPU / 16 GiB) plus worker-3 (2 vCPU / 8 GiB) on shushou, as microVMs. They come with a genK3sAgentModule and the per-host/output/home-manager wiring, and are labelled node-role.kubernetes.io/worker=true. The masters keep their control-plane:NoSchedule taint, so workloads land on the workers. --- home/hosts/linux/k3s-test-1-master-2.nix | 3 ++ home/hosts/linux/k3s-test-1-master-3.nix | 3 ++ home/hosts/linux/k3s-test-1-worker-1.nix | 3 ++ home/hosts/linux/k3s-test-1-worker-2.nix | 3 ++ home/hosts/linux/k3s-test-1-worker-3.nix | 3 ++ hosts/12kingdoms-shoryu/microvm.nix | 25 +++++++++- hosts/12kingdoms-shushou/microvm.nix | 48 ++++++++++++++++++- hosts/README.md | 3 +- hosts/k8s/README.md | 9 +++- hosts/k8s/k3s-test-1-worker-1/default.nix | 48 +++++++++++++++++++ hosts/k8s/k3s-test-1-worker-2/default.nix | 48 +++++++++++++++++++ hosts/k8s/k3s-test-1-worker-3/default.nix | 48 +++++++++++++++++++ lib/default.nix | 1 + lib/genK3sAgentModule.nix | 45 +++++++++++++++++ .../x86_64-linux/src/k3s-test-1-worker-1.nix | 40 ++++++++++++++++ .../x86_64-linux/src/k3s-test-1-worker-2.nix | 40 ++++++++++++++++ .../x86_64-linux/src/k3s-test-1-worker-3.nix | 40 ++++++++++++++++ 17 files changed, 406 insertions(+), 4 deletions(-) create mode 100644 home/hosts/linux/k3s-test-1-master-2.nix create mode 100644 home/hosts/linux/k3s-test-1-master-3.nix create mode 100644 home/hosts/linux/k3s-test-1-worker-1.nix create mode 100644 home/hosts/linux/k3s-test-1-worker-2.nix create mode 100644 home/hosts/linux/k3s-test-1-worker-3.nix create mode 100644 hosts/k8s/k3s-test-1-worker-1/default.nix create mode 100644 hosts/k8s/k3s-test-1-worker-2/default.nix create mode 100644 hosts/k8s/k3s-test-1-worker-3/default.nix create mode 100644 lib/genK3sAgentModule.nix create mode 100644 outputs/x86_64-linux/src/k3s-test-1-worker-1.nix create mode 100644 outputs/x86_64-linux/src/k3s-test-1-worker-2.nix create mode 100644 outputs/x86_64-linux/src/k3s-test-1-worker-3.nix diff --git a/home/hosts/linux/k3s-test-1-master-2.nix b/home/hosts/linux/k3s-test-1-master-2.nix new file mode 100644 index 00000000..a67103bf --- /dev/null +++ b/home/hosts/linux/k3s-test-1-master-2.nix @@ -0,0 +1,3 @@ +{ + imports = [ ../../linux/core.nix ]; +} diff --git a/home/hosts/linux/k3s-test-1-master-3.nix b/home/hosts/linux/k3s-test-1-master-3.nix new file mode 100644 index 00000000..a67103bf --- /dev/null +++ b/home/hosts/linux/k3s-test-1-master-3.nix @@ -0,0 +1,3 @@ +{ + imports = [ ../../linux/core.nix ]; +} diff --git a/home/hosts/linux/k3s-test-1-worker-1.nix b/home/hosts/linux/k3s-test-1-worker-1.nix new file mode 100644 index 00000000..a67103bf --- /dev/null +++ b/home/hosts/linux/k3s-test-1-worker-1.nix @@ -0,0 +1,3 @@ +{ + imports = [ ../../linux/core.nix ]; +} diff --git a/home/hosts/linux/k3s-test-1-worker-2.nix b/home/hosts/linux/k3s-test-1-worker-2.nix new file mode 100644 index 00000000..a67103bf --- /dev/null +++ b/home/hosts/linux/k3s-test-1-worker-2.nix @@ -0,0 +1,3 @@ +{ + imports = [ ../../linux/core.nix ]; +} diff --git a/home/hosts/linux/k3s-test-1-worker-3.nix b/home/hosts/linux/k3s-test-1-worker-3.nix new file mode 100644 index 00000000..a67103bf --- /dev/null +++ b/home/hosts/linux/k3s-test-1-worker-3.nix @@ -0,0 +1,3 @@ +{ + imports = [ ../../linux/core.nix ]; +} diff --git a/hosts/12kingdoms-shoryu/microvm.nix b/hosts/12kingdoms-shoryu/microvm.nix index acbc92db..53b5172c 100644 --- a/hosts/12kingdoms-shoryu/microvm.nix +++ b/hosts/12kingdoms-shoryu/microvm.nix @@ -24,8 +24,23 @@ config.imports = [ ../k8s/k3s-test-1-master-1 ]; }; + microvm.vms.k3s-test-1-worker-1 = { + autostart = true; + restartIfChanged = true; + specialArgs = { + inherit + myvars + mylib + agenix + mysecrets + ; + }; + config.imports = [ ../k8s/k3s-test-1-worker-1 ]; + }; + # Attach the guest's tap to the VM bridge, the same way the physical NIC is - # attached. The tap name is derived from the guest IP (192.168.5.114 -> vm114). + # attached. The tap name is derived from the guest IP (192.168.5.114 -> vm114, + # 192.168.5.111 -> vm111), as IFNAMSIZ caps interface names at 15 characters. systemd.network.networks."20-vm114" = { matchConfig.Name = [ "vm114" ]; networkConfig = { @@ -34,4 +49,12 @@ }; linkConfig.RequiredForOnline = "no"; }; + systemd.network.networks."20-vm111" = { + matchConfig.Name = [ "vm111" ]; + networkConfig = { + LinkLocalAddressing = "no"; + Bridge = "br0"; + }; + linkConfig.RequiredForOnline = "no"; + }; } diff --git a/hosts/12kingdoms-shushou/microvm.nix b/hosts/12kingdoms-shushou/microvm.nix index 4243fcf9..006e515d 100644 --- a/hosts/12kingdoms-shushou/microvm.nix +++ b/hosts/12kingdoms-shushou/microvm.nix @@ -24,8 +24,38 @@ config.imports = [ ../k8s/k3s-test-1-master-3 ]; }; + microvm.vms.k3s-test-1-worker-2 = { + autostart = true; + restartIfChanged = true; + specialArgs = { + inherit + myvars + mylib + agenix + mysecrets + ; + }; + config.imports = [ ../k8s/k3s-test-1-worker-2 ]; + }; + + microvm.vms.k3s-test-1-worker-3 = { + autostart = true; + restartIfChanged = true; + specialArgs = { + inherit + myvars + mylib + agenix + mysecrets + ; + }; + config.imports = [ ../k8s/k3s-test-1-worker-3 ]; + }; + # Attach the guest's tap to the VM bridge, the same way the physical NIC is - # attached. The tap name is derived from the guest IP (192.168.5.116 -> vm116). + # attached. The tap name is derived from the guest IP (192.168.5.116 -> vm116, + # 192.168.5.112 -> vm112, 192.168.5.113 -> vm113), as IFNAMSIZ caps interface + # names at 15 characters. systemd.network.networks."20-vm116" = { matchConfig.Name = [ "vm116" ]; networkConfig = { @@ -34,4 +64,20 @@ }; linkConfig.RequiredForOnline = "no"; }; + systemd.network.networks."20-vm112" = { + matchConfig.Name = [ "vm112" ]; + networkConfig = { + LinkLocalAddressing = "no"; + Bridge = "br0"; + }; + linkConfig.RequiredForOnline = "no"; + }; + systemd.network.networks."20-vm113" = { + matchConfig.Name = [ "vm113" ]; + networkConfig = { + LinkLocalAddressing = "no"; + Bridge = "br0"; + }; + linkConfig.RequiredForOnline = "no"; + }; } diff --git a/hosts/README.md b/hosts/README.md index 82a2e8aa..b42b1cbd 100644 --- a/hosts/README.md +++ b/hosts/README.md @@ -46,7 +46,8 @@ Named after "Twelve Kingdoms": #### `k8s` - Kubernetes Infrastructure - **VM Cluster**: 3 physical mini PCs (shoryu, shushou, youko) running all VMs -- **K3s Testing**: `k3s-test-1-master-{1,2,3}`, running as microVMs on those hosts +- **K3s Testing**: `k3s-test-1-master-{1,2,3}` (control plane) + `k3s-test-1-worker-{1,2,3}` + (workloads; two on `shushou`, one on `shoryu`), running as microVMs ### External Systems diff --git a/hosts/k8s/README.md b/hosts/k8s/README.md index 8fbe8826..e4232741 100644 --- a/hosts/k8s/README.md +++ b/hosts/k8s/README.md @@ -30,7 +30,14 @@ To move an existing VM over in place, copy the host key and `machine-id` from it `etc.img`, and `/var/lib/rancher/k3s/server` into `var.img`, **before the first boot** — then it keeps its identity and etcd membership instead of bootstrapping a new cluster. -1. `k3s-test-1-master-{1,2,3}` — running as microVMs +1. `k3s-test-1-master-{1,2,3}` — control plane, running as microVMs; tainted + `node-role.kubernetes.io/control-plane:NoSchedule` +1. `k3s-test-1-worker-{1,2,3}` — workloads, running as microVMs + (`node-role.kubernetes.io/worker=true`) + +Placement: `worker-1` (4 vCPU / 16 GiB) runs on `shoryu`; `worker-2` (4 vCPU / 16 GiB) and +`worker-3` (2 vCPU / 8 GiB) run on `shushou`. `youko` has no worker because it has the least free +memory and also runs the homelab services. ## TODO / Known issues diff --git a/hosts/k8s/k3s-test-1-worker-1/default.nix b/hosts/k8s/k3s-test-1-worker-1/default.nix new file mode 100644 index 00000000..0edca784 --- /dev/null +++ b/hosts/k8s/k3s-test-1-worker-1/default.nix @@ -0,0 +1,48 @@ +{ + config, + pkgs, + lib, + myvars, + mylib, + ... +}: +let + hostName = "k3s-test-1-worker-1"; # Define your hostname. + + coreModule = mylib.genMicrovmGuestModule { + inherit pkgs hostName; + inherit (myvars) networking; + vcpu = 4; + mem = 16384; + varSize = 20480; + }; + k3sModule = mylib.genK3sAgentModule { + inherit pkgs; + tokenFile = config.age.secrets."k3s-test-1-token".path; + # use my own domain & kube-vip's virtual IP for the API server + # so that the API server can always be accessed even if some nodes are down + masterHost = "test-cluster-1.writefor.fun"; + # workloads run on the workers; the masters are tainted (see k3s-test-1-master-*) + nodeLabels = [ "node-role.kubernetes.io/worker=true" ]; + }; +in +{ + imports = + (mylib.scanPaths ./.) + ++ (map mylib.relativeToRoot [ + "secrets/nixos.nix" + "modules/nixos/server/server.nix" + ]) + ++ [ + coreModule + k3sModule + ]; + + # k3s-test-1-token lives in this secret group + modules.secrets.server.kubernetes.enable = true; + + # The guest reuses the host's already-instantiated nixpkgs (shared store), + # so nixpkgs.config may not be set. The shared base modules set + # nixpkgs.config.allowUnfree, so force it empty here. + nixpkgs.config = lib.mkForce { }; +} diff --git a/hosts/k8s/k3s-test-1-worker-2/default.nix b/hosts/k8s/k3s-test-1-worker-2/default.nix new file mode 100644 index 00000000..a6cf3986 --- /dev/null +++ b/hosts/k8s/k3s-test-1-worker-2/default.nix @@ -0,0 +1,48 @@ +{ + config, + pkgs, + lib, + myvars, + mylib, + ... +}: +let + hostName = "k3s-test-1-worker-2"; # Define your hostname. + + coreModule = mylib.genMicrovmGuestModule { + inherit pkgs hostName; + inherit (myvars) networking; + vcpu = 4; + mem = 16384; + varSize = 20480; + }; + k3sModule = mylib.genK3sAgentModule { + inherit pkgs; + tokenFile = config.age.secrets."k3s-test-1-token".path; + # use my own domain & kube-vip's virtual IP for the API server + # so that the API server can always be accessed even if some nodes are down + masterHost = "test-cluster-1.writefor.fun"; + # workloads run on the workers; the masters are tainted (see k3s-test-1-master-*) + nodeLabels = [ "node-role.kubernetes.io/worker=true" ]; + }; +in +{ + imports = + (mylib.scanPaths ./.) + ++ (map mylib.relativeToRoot [ + "secrets/nixos.nix" + "modules/nixos/server/server.nix" + ]) + ++ [ + coreModule + k3sModule + ]; + + # k3s-test-1-token lives in this secret group + modules.secrets.server.kubernetes.enable = true; + + # The guest reuses the host's already-instantiated nixpkgs (shared store), + # so nixpkgs.config may not be set. The shared base modules set + # nixpkgs.config.allowUnfree, so force it empty here. + nixpkgs.config = lib.mkForce { }; +} diff --git a/hosts/k8s/k3s-test-1-worker-3/default.nix b/hosts/k8s/k3s-test-1-worker-3/default.nix new file mode 100644 index 00000000..95de1903 --- /dev/null +++ b/hosts/k8s/k3s-test-1-worker-3/default.nix @@ -0,0 +1,48 @@ +{ + config, + pkgs, + lib, + myvars, + mylib, + ... +}: +let + hostName = "k3s-test-1-worker-3"; # Define your hostname. + + coreModule = mylib.genMicrovmGuestModule { + inherit pkgs hostName; + inherit (myvars) networking; + vcpu = 2; + mem = 8192; + varSize = 20480; + }; + k3sModule = mylib.genK3sAgentModule { + inherit pkgs; + tokenFile = config.age.secrets."k3s-test-1-token".path; + # use my own domain & kube-vip's virtual IP for the API server + # so that the API server can always be accessed even if some nodes are down + masterHost = "test-cluster-1.writefor.fun"; + # workloads run on the workers; the masters are tainted (see k3s-test-1-master-*) + nodeLabels = [ "node-role.kubernetes.io/worker=true" ]; + }; +in +{ + imports = + (mylib.scanPaths ./.) + ++ (map mylib.relativeToRoot [ + "secrets/nixos.nix" + "modules/nixos/server/server.nix" + ]) + ++ [ + coreModule + k3sModule + ]; + + # k3s-test-1-token lives in this secret group + modules.secrets.server.kubernetes.enable = true; + + # The guest reuses the host's already-instantiated nixpkgs (shared store), + # so nixpkgs.config may not be set. The shared base modules set + # nixpkgs.config.allowUnfree, so force it empty here. + nixpkgs.config = lib.mkForce { }; +} diff --git a/lib/default.nix b/lib/default.nix index 96d2b075..3e6fc894 100644 --- a/lib/default.nix +++ b/lib/default.nix @@ -7,6 +7,7 @@ attrs = import ./attrs.nix { inherit lib; }; genK3sServerModule = import ./genK3sServerModule.nix; + genK3sAgentModule = import ./genK3sAgentModule.nix; genVmHostModule = import ./genVmHostModule.nix; genMicrovmGuestModule = import ./genMicrovmGuestModule.nix; diff --git a/lib/genK3sAgentModule.nix b/lib/genK3sAgentModule.nix new file mode 100644 index 00000000..e4f28095 --- /dev/null +++ b/lib/genK3sAgentModule.nix @@ -0,0 +1,45 @@ +{ + pkgs, + masterHost, + tokenFile, + nodeLabels ? [ ], + k3sExtraArgs ? [ ], + ... +}: +let + package = pkgs.k3s; +in +{ + environment.systemPackages = [ package ]; + + # Kernel modules required by cilium + boot.kernelModules = [ + "ip6_tables" + "ip6table_mangle" + "ip6table_raw" + "ip6table_filter" + ]; + networking.enableIPv6 = true; + networking.nat = { + enable = true; + enableIPv6 = true; + }; + + services.k3s = { + enable = true; + inherit package tokenFile; + + role = "agent"; + serverAddr = "https://${masterHost}:6443"; + # https://docs.k3s.io/cli/agent + extraFlags = + let + flagList = [ + "--data-dir /var/lib/rancher/k3s" + ] + ++ (map (label: "--node-label=${label}") nodeLabels) + ++ k3sExtraArgs; + in + pkgs.lib.concatStringsSep " " flagList; + }; +} diff --git a/outputs/x86_64-linux/src/k3s-test-1-worker-1.nix b/outputs/x86_64-linux/src/k3s-test-1-worker-1.nix new file mode 100644 index 00000000..92181322 --- /dev/null +++ b/outputs/x86_64-linux/src/k3s-test-1-worker-1.nix @@ -0,0 +1,40 @@ +{ + # NOTE: the args not used in this file CAN NOT be removed! + # because haumea pass argument lazily, + # and these arguments are used in the functions like `mylib.nixosSystem`, `mylib.colmenaSystem`, etc. + inputs, + lib, + mylib, + myvars, + system, + genSpecialArgs, + ... +}@args: +let + name = "k3s-test-1-worker-1"; + tags = [ name ]; + ssh-user = "root"; + + modules = { + # the host dir is the complete config (it pulls in the shared modules) + nixos-modules = + (map mylib.relativeToRoot [ + "hosts/k8s/${name}" + ]) + ++ [ + # this runs as a MicroVM + inputs.microvm.nixosModules.microvm + { modules.secrets.server.kubernetes.enable = true; } + ]; + home-modules = map mylib.relativeToRoot [ + "home/hosts/linux/${name}.nix" + ]; + }; + + systemArgs = modules // args; +in +{ + nixosConfigurations.${name} = mylib.nixosSystem systemArgs; + + colmena.${name} = mylib.colmenaSystem (systemArgs // { inherit tags ssh-user; }); +} diff --git a/outputs/x86_64-linux/src/k3s-test-1-worker-2.nix b/outputs/x86_64-linux/src/k3s-test-1-worker-2.nix new file mode 100644 index 00000000..424de015 --- /dev/null +++ b/outputs/x86_64-linux/src/k3s-test-1-worker-2.nix @@ -0,0 +1,40 @@ +{ + # NOTE: the args not used in this file CAN NOT be removed! + # because haumea pass argument lazily, + # and these arguments are used in the functions like `mylib.nixosSystem`, `mylib.colmenaSystem`, etc. + inputs, + lib, + mylib, + myvars, + system, + genSpecialArgs, + ... +}@args: +let + name = "k3s-test-1-worker-2"; + tags = [ name ]; + ssh-user = "root"; + + modules = { + # the host dir is the complete config (it pulls in the shared modules) + nixos-modules = + (map mylib.relativeToRoot [ + "hosts/k8s/${name}" + ]) + ++ [ + # this runs as a MicroVM + inputs.microvm.nixosModules.microvm + { modules.secrets.server.kubernetes.enable = true; } + ]; + home-modules = map mylib.relativeToRoot [ + "home/hosts/linux/${name}.nix" + ]; + }; + + systemArgs = modules // args; +in +{ + nixosConfigurations.${name} = mylib.nixosSystem systemArgs; + + colmena.${name} = mylib.colmenaSystem (systemArgs // { inherit tags ssh-user; }); +} diff --git a/outputs/x86_64-linux/src/k3s-test-1-worker-3.nix b/outputs/x86_64-linux/src/k3s-test-1-worker-3.nix new file mode 100644 index 00000000..977a0f9a --- /dev/null +++ b/outputs/x86_64-linux/src/k3s-test-1-worker-3.nix @@ -0,0 +1,40 @@ +{ + # NOTE: the args not used in this file CAN NOT be removed! + # because haumea pass argument lazily, + # and these arguments are used in the functions like `mylib.nixosSystem`, `mylib.colmenaSystem`, etc. + inputs, + lib, + mylib, + myvars, + system, + genSpecialArgs, + ... +}@args: +let + name = "k3s-test-1-worker-3"; + tags = [ name ]; + ssh-user = "root"; + + modules = { + # the host dir is the complete config (it pulls in the shared modules) + nixos-modules = + (map mylib.relativeToRoot [ + "hosts/k8s/${name}" + ]) + ++ [ + # this runs as a MicroVM + inputs.microvm.nixosModules.microvm + { modules.secrets.server.kubernetes.enable = true; } + ]; + home-modules = map mylib.relativeToRoot [ + "home/hosts/linux/${name}.nix" + ]; + }; + + systemArgs = modules // args; +in +{ + nixosConfigurations.${name} = mylib.nixosSystem systemArgs; + + colmena.${name} = mylib.colmenaSystem (systemArgs // { inherit tags ssh-user; }); +}