From d40ed42f6ccb8f43f02dc547fa4430be35842f9e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?ryan4yin=20=7C=20=E4=BA=8C=E8=8A=B1?= Date: Mon, 21 Sep 2026 12:56:49 +0800 Subject: [PATCH] feat(k3s-test-1): add three worker nodes and taint the control plane (#283) * feat(k3s-test-1): add three worker nodes and taint the control plane - hosts/k8s/k3s-test-1-worker-{1,2,3} (k3s agents) + flake outputs - vars/networking: .117/.118/.119 (the .111-.113 range is still held by the never-deployed k3s-prod-1 workers) - k3s-test-1-master-{1,2,3}: nodeTaints=node-role.kubernetes.io/control-plane:NoSchedule - Justfile: upload the worker images too * chore(k3s-test-1): move the workers to .111-.113 The k3s-prod-1 entries that held those addresses are gone (nix-config#284). --- Justfile | 3 ++ hosts/k8s/k3s-test-1-master-1/default.nix | 3 ++ hosts/k8s/k3s-test-1-master-2/default.nix | 3 ++ hosts/k8s/k3s-test-1-master-3/default.nix | 3 ++ hosts/k8s/k3s-test-1-worker-1/default.nix | 30 ++++++++++++++ hosts/k8s/k3s-test-1-worker-2/default.nix | 30 ++++++++++++++ hosts/k8s/k3s-test-1-worker-3/default.nix | 30 ++++++++++++++ .../x86_64-linux/src/k3s-test-1-worker-1.nix | 41 +++++++++++++++++++ .../x86_64-linux/src/k3s-test-1-worker-2.nix | 41 +++++++++++++++++++ .../x86_64-linux/src/k3s-test-1-worker-3.nix | 41 +++++++++++++++++++ vars/networking.nix | 15 +++++++ 11 files changed, 240 insertions(+) create mode 100644 hosts/k8s/k3s-test-1-worker-1/default.nix create mode 100644 hosts/k8s/k3s-test-1-worker-2/default.nix create mode 100644 hosts/k8s/k3s-test-1-worker-3/default.nix create mode 100644 outputs/x86_64-linux/src/k3s-test-1-worker-1.nix create mode 100644 outputs/x86_64-linux/src/k3s-test-1-worker-2.nix create mode 100644 outputs/x86_64-linux/src/k3s-test-1-worker-3.nix diff --git a/Justfile b/Justfile index 1b1d3d4c..819d2ea0 100644 --- a/Justfile +++ b/Justfile @@ -264,6 +264,9 @@ upload-k3s-test verbosity="normal": upload-vm k3s-test-1-master-1 {{verbosity}}; upload-vm k3s-test-1-master-2 {{verbosity}}; upload-vm k3s-test-1-master-3 {{verbosity}}; + upload-vm k3s-test-1-worker-1 {{verbosity}}; + upload-vm k3s-test-1-worker-2 {{verbosity}}; + upload-vm k3s-test-1-worker-3 {{verbosity}}; [linux] [group('homelab')] diff --git a/hosts/k8s/k3s-test-1-master-1/default.nix b/hosts/k8s/k3s-test-1-master-1/default.nix index 2a5c8ab6..f9a5f87b 100644 --- a/hosts/k8s/k3s-test-1-master-1/default.nix +++ b/hosts/k8s/k3s-test-1-master-1/default.nix @@ -21,6 +21,9 @@ let # use my own domain & kube-vip's virtual IP for the API server # so that the API server can always be accessed even if some nodes are down masterHost = "test-cluster-1.writefor.fun"; + # the control plane only runs the control plane; workloads run on the + # k3s-test-1-worker-* nodes (labelled node-role.kubernetes.io/worker=true) + nodeTaints = [ "node-role.kubernetes.io/control-plane:NoSchedule" ]; # k3sExtraArgs = [ # # IPv4 Private CIDR(full) - 172.16.0.0/12 diff --git a/hosts/k8s/k3s-test-1-master-2/default.nix b/hosts/k8s/k3s-test-1-master-2/default.nix index 65661b57..8f60aa7d 100644 --- a/hosts/k8s/k3s-test-1-master-2/default.nix +++ b/hosts/k8s/k3s-test-1-master-2/default.nix @@ -19,6 +19,9 @@ let # use my own domain & kube-vip's virtual IP for the API server # so that the API server can always be accessed even if some nodes are down masterHost = "test-cluster-1.writefor.fun"; + # the control plane only runs the control plane; workloads run on the + # k3s-test-1-worker-* nodes (labelled node-role.kubernetes.io/worker=true) + nodeTaints = [ "node-role.kubernetes.io/control-plane:NoSchedule" ]; # k3sExtraArgs = [ # # IPv4 Private CIDR(full) - 172.16.0.0/12 diff --git a/hosts/k8s/k3s-test-1-master-3/default.nix b/hosts/k8s/k3s-test-1-master-3/default.nix index 483777fa..d11e1e7b 100644 --- a/hosts/k8s/k3s-test-1-master-3/default.nix +++ b/hosts/k8s/k3s-test-1-master-3/default.nix @@ -19,6 +19,9 @@ let # use my own domain & kube-vip's virtual IP for the API server # so that the API server can always be accessed even if some nodes are down masterHost = "test-cluster-1.writefor.fun"; + # the control plane only runs the control plane; workloads run on the + # k3s-test-1-worker-* nodes (labelled node-role.kubernetes.io/worker=true) + nodeTaints = [ "node-role.kubernetes.io/control-plane:NoSchedule" ]; # k3sExtraArgs = [ # # IPv4 Private CIDR(full) - 172.16.0.0/12 diff --git a/hosts/k8s/k3s-test-1-worker-1/default.nix b/hosts/k8s/k3s-test-1-worker-1/default.nix new file mode 100644 index 00000000..1c0dd693 --- /dev/null +++ b/hosts/k8s/k3s-test-1-worker-1/default.nix @@ -0,0 +1,30 @@ +{ + config, + pkgs, + myvars, + mylib, + ... +}: +let + hostName = "k3s-test-1-worker-1"; # define your hostname. + + coreModule = mylib.genKubeVirtGuestModule { + inherit pkgs hostName; + inherit (myvars) networking; + }; + k3sModule = mylib.genK3sAgentModule { + inherit pkgs; + tokenFile = config.age.secrets."k3s-test-1-token".path; + # use my own domain & kube-vip's virtual IP for the API server + # so that the API server can always be accessed even if some nodes are down + masterHost = "test-cluster-1.writefor.fun"; + # workloads run on the workers; the masters are tainted (see k3s-test-1-master-*) + nodeLabels = [ "node-role.kubernetes.io/worker=true" ]; + }; +in +{ + imports = (mylib.scanPaths ./.) ++ [ + coreModule + k3sModule + ]; +} diff --git a/hosts/k8s/k3s-test-1-worker-2/default.nix b/hosts/k8s/k3s-test-1-worker-2/default.nix new file mode 100644 index 00000000..7fd24045 --- /dev/null +++ b/hosts/k8s/k3s-test-1-worker-2/default.nix @@ -0,0 +1,30 @@ +{ + config, + pkgs, + myvars, + mylib, + ... +}: +let + hostName = "k3s-test-1-worker-2"; # define your hostname. + + coreModule = mylib.genKubeVirtGuestModule { + inherit pkgs hostName; + inherit (myvars) networking; + }; + k3sModule = mylib.genK3sAgentModule { + inherit pkgs; + tokenFile = config.age.secrets."k3s-test-1-token".path; + # use my own domain & kube-vip's virtual IP for the API server + # so that the API server can always be accessed even if some nodes are down + masterHost = "test-cluster-1.writefor.fun"; + # workloads run on the workers; the masters are tainted (see k3s-test-1-master-*) + nodeLabels = [ "node-role.kubernetes.io/worker=true" ]; + }; +in +{ + imports = (mylib.scanPaths ./.) ++ [ + coreModule + k3sModule + ]; +} diff --git a/hosts/k8s/k3s-test-1-worker-3/default.nix b/hosts/k8s/k3s-test-1-worker-3/default.nix new file mode 100644 index 00000000..1352b10e --- /dev/null +++ b/hosts/k8s/k3s-test-1-worker-3/default.nix @@ -0,0 +1,30 @@ +{ + config, + pkgs, + myvars, + mylib, + ... +}: +let + hostName = "k3s-test-1-worker-3"; # define your hostname. + + coreModule = mylib.genKubeVirtGuestModule { + inherit pkgs hostName; + inherit (myvars) networking; + }; + k3sModule = mylib.genK3sAgentModule { + inherit pkgs; + tokenFile = config.age.secrets."k3s-test-1-token".path; + # use my own domain & kube-vip's virtual IP for the API server + # so that the API server can always be accessed even if some nodes are down + masterHost = "test-cluster-1.writefor.fun"; + # workloads run on the workers; the masters are tainted (see k3s-test-1-master-*) + nodeLabels = [ "node-role.kubernetes.io/worker=true" ]; + }; +in +{ + imports = (mylib.scanPaths ./.) ++ [ + coreModule + k3sModule + ]; +} diff --git a/outputs/x86_64-linux/src/k3s-test-1-worker-1.nix b/outputs/x86_64-linux/src/k3s-test-1-worker-1.nix new file mode 100644 index 00000000..c345d756 --- /dev/null +++ b/outputs/x86_64-linux/src/k3s-test-1-worker-1.nix @@ -0,0 +1,41 @@ +{ + # NOTE: the args not used in this file CAN NOT be removed! + # because haumea pass argument lazily, + # and these arguments are used in the functions like `mylib.nixosSystem`, `mylib.colmenaSystem`, etc. + inputs, + lib, + mylib, + myvars, + system, + genSpecialArgs, + ... +}@args: +let + name = "k3s-test-1-worker-1"; + tags = [ name ]; + ssh-user = "root"; + + modules = { + nixos-modules = + (map mylib.relativeToRoot [ + # common + "secrets/nixos.nix" + "modules/nixos/server/server.nix" + "modules/nixos/server/kubevirt-hardware-configuration.nix" + # host specific + "hosts/k8s/${name}" + ]) + ++ [ + { modules.secrets.server.kubernetes.enable = true; } + ]; + }; + + systemArgs = modules // args; +in +{ + nixosConfigurations.${name} = mylib.nixosSystem systemArgs; + + colmena.${name} = mylib.colmenaSystem (systemArgs // { inherit tags ssh-user; }); + + packages.${name} = inputs.self.nixosConfigurations.${name}.config.system.build.images.kubevirt; +} diff --git a/outputs/x86_64-linux/src/k3s-test-1-worker-2.nix b/outputs/x86_64-linux/src/k3s-test-1-worker-2.nix new file mode 100644 index 00000000..089cc112 --- /dev/null +++ b/outputs/x86_64-linux/src/k3s-test-1-worker-2.nix @@ -0,0 +1,41 @@ +{ + # NOTE: the args not used in this file CAN NOT be removed! + # because haumea pass argument lazily, + # and these arguments are used in the functions like `mylib.nixosSystem`, `mylib.colmenaSystem`, etc. + inputs, + lib, + mylib, + myvars, + system, + genSpecialArgs, + ... +}@args: +let + name = "k3s-test-1-worker-2"; + tags = [ name ]; + ssh-user = "root"; + + modules = { + nixos-modules = + (map mylib.relativeToRoot [ + # common + "secrets/nixos.nix" + "modules/nixos/server/server.nix" + "modules/nixos/server/kubevirt-hardware-configuration.nix" + # host specific + "hosts/k8s/${name}" + ]) + ++ [ + { modules.secrets.server.kubernetes.enable = true; } + ]; + }; + + systemArgs = modules // args; +in +{ + nixosConfigurations.${name} = mylib.nixosSystem systemArgs; + + colmena.${name} = mylib.colmenaSystem (systemArgs // { inherit tags ssh-user; }); + + packages.${name} = inputs.self.nixosConfigurations.${name}.config.system.build.images.kubevirt; +} diff --git a/outputs/x86_64-linux/src/k3s-test-1-worker-3.nix b/outputs/x86_64-linux/src/k3s-test-1-worker-3.nix new file mode 100644 index 00000000..06dac6c0 --- /dev/null +++ b/outputs/x86_64-linux/src/k3s-test-1-worker-3.nix @@ -0,0 +1,41 @@ +{ + # NOTE: the args not used in this file CAN NOT be removed! + # because haumea pass argument lazily, + # and these arguments are used in the functions like `mylib.nixosSystem`, `mylib.colmenaSystem`, etc. + inputs, + lib, + mylib, + myvars, + system, + genSpecialArgs, + ... +}@args: +let + name = "k3s-test-1-worker-3"; + tags = [ name ]; + ssh-user = "root"; + + modules = { + nixos-modules = + (map mylib.relativeToRoot [ + # common + "secrets/nixos.nix" + "modules/nixos/server/server.nix" + "modules/nixos/server/kubevirt-hardware-configuration.nix" + # host specific + "hosts/k8s/${name}" + ]) + ++ [ + { modules.secrets.server.kubernetes.enable = true; } + ]; + }; + + systemArgs = modules // args; +in +{ + nixosConfigurations.${name} = mylib.nixosSystem systemArgs; + + colmena.${name} = mylib.colmenaSystem (systemArgs // { inherit tags ssh-user; }); + + packages.${name} = inputs.self.nixosConfigurations.${name}.config.system.build.images.kubevirt; +} diff --git a/vars/networking.nix b/vars/networking.nix index e6a48673..f37208f0 100644 --- a/vars/networking.nix +++ b/vars/networking.nix @@ -119,6 +119,21 @@ rec { iface = "enp2s0"; ipv4 = "192.168.5.116"; }; + k3s-test-1-worker-1 = { + # KubeVirt VM + iface = "enp2s0"; + ipv4 = "192.168.5.111"; + }; + k3s-test-1-worker-2 = { + # KubeVirt VM + iface = "enp2s0"; + ipv4 = "192.168.5.112"; + }; + k3s-test-1-worker-3 = { + # KubeVirt VM + iface = "enp2s0"; + ipv4 = "192.168.5.113"; + }; }; hostsInterface = lib.attrsets.mapAttrs (key: val: {