feat(k3s-test-1): add three worker nodes and taint the control plane (#283)

* feat(k3s-test-1): add three worker nodes and taint the control plane

- hosts/k8s/k3s-test-1-worker-{1,2,3} (k3s agents) + flake outputs
- vars/networking: .117/.118/.119 (the .111-.113 range is still held by the
  never-deployed k3s-prod-1 workers)
- k3s-test-1-master-{1,2,3}: nodeTaints=node-role.kubernetes.io/control-plane:NoSchedule
- Justfile: upload the worker images too

* chore(k3s-test-1): move the workers to .111-.113

The k3s-prod-1 entries that held those addresses are gone (nix-config#284).

Justfile

@@ -264,6 +264,9 @@ upload-k3s-test verbosity="normal":
   upload-vm k3s-test-1-master-1 {{verbosity}};
   upload-vm k3s-test-1-master-2 {{verbosity}};
   upload-vm k3s-test-1-master-3 {{verbosity}};
+  upload-vm k3s-test-1-worker-1 {{verbosity}};
+  upload-vm k3s-test-1-worker-2 {{verbosity}};
+  upload-vm k3s-test-1-worker-3 {{verbosity}};
 
 [linux]
 [group('homelab')]

hosts/k8s/k3s-test-1-master-1/default.nix

@@ -21,6 +21,9 @@ let
     # use my own domain & kube-vip's virtual IP for the API server
     # so that the API server can always be accessed even if some nodes are down
     masterHost = "test-cluster-1.writefor.fun";
+    # the control plane only runs the control plane; workloads run on the
+    # k3s-test-1-worker-* nodes (labelled node-role.kubernetes.io/worker=true)
+    nodeTaints = [ "node-role.kubernetes.io/control-plane:NoSchedule" ];
 
     # k3sExtraArgs = [
     #   # IPv4 Private CIDR(full) - 172.16.0.0/12

hosts/k8s/k3s-test-1-master-2/default.nix

@@ -19,6 +19,9 @@ let
     # use my own domain & kube-vip's virtual IP for the API server
     # so that the API server can always be accessed even if some nodes are down
     masterHost = "test-cluster-1.writefor.fun";
+    # the control plane only runs the control plane; workloads run on the
+    # k3s-test-1-worker-* nodes (labelled node-role.kubernetes.io/worker=true)
+    nodeTaints = [ "node-role.kubernetes.io/control-plane:NoSchedule" ];
 
     # k3sExtraArgs = [
     #   # IPv4 Private CIDR(full) - 172.16.0.0/12

hosts/k8s/k3s-test-1-master-3/default.nix

@@ -19,6 +19,9 @@ let
     # use my own domain & kube-vip's virtual IP for the API server
     # so that the API server can always be accessed even if some nodes are down
     masterHost = "test-cluster-1.writefor.fun";
+    # the control plane only runs the control plane; workloads run on the
+    # k3s-test-1-worker-* nodes (labelled node-role.kubernetes.io/worker=true)
+    nodeTaints = [ "node-role.kubernetes.io/control-plane:NoSchedule" ];
 
     # k3sExtraArgs = [
     #   # IPv4 Private CIDR(full) - 172.16.0.0/12

hosts/k8s/k3s-test-1-worker-1/default.nix

@@ -0,0 +1,30 @@
+{
+  config,
+  pkgs,
+  myvars,
+  mylib,
+  ...
+}:
+let
+  hostName = "k3s-test-1-worker-1"; # define your hostname.
+
+  coreModule = mylib.genKubeVirtGuestModule {
+    inherit pkgs hostName;
+    inherit (myvars) networking;
+  };
+  k3sModule = mylib.genK3sAgentModule {
+    inherit pkgs;
+    tokenFile = config.age.secrets."k3s-test-1-token".path;
+    # use my own domain & kube-vip's virtual IP for the API server
+    # so that the API server can always be accessed even if some nodes are down
+    masterHost = "test-cluster-1.writefor.fun";
+    # workloads run on the workers; the masters are tainted (see k3s-test-1-master-*)
+    nodeLabels = [ "node-role.kubernetes.io/worker=true" ];
+  };
+in
+{
+  imports = (mylib.scanPaths ./.) ++ [
+    coreModule
+    k3sModule
+  ];
+}

hosts/k8s/k3s-test-1-worker-2/default.nix

@@ -0,0 +1,30 @@
+{
+  config,
+  pkgs,
+  myvars,
+  mylib,
+  ...
+}:
+let
+  hostName = "k3s-test-1-worker-2"; # define your hostname.
+
+  coreModule = mylib.genKubeVirtGuestModule {
+    inherit pkgs hostName;
+    inherit (myvars) networking;
+  };
+  k3sModule = mylib.genK3sAgentModule {
+    inherit pkgs;
+    tokenFile = config.age.secrets."k3s-test-1-token".path;
+    # use my own domain & kube-vip's virtual IP for the API server
+    # so that the API server can always be accessed even if some nodes are down
+    masterHost = "test-cluster-1.writefor.fun";
+    # workloads run on the workers; the masters are tainted (see k3s-test-1-master-*)
+    nodeLabels = [ "node-role.kubernetes.io/worker=true" ];
+  };
+in
+{
+  imports = (mylib.scanPaths ./.) ++ [
+    coreModule
+    k3sModule
+  ];
+}

hosts/k8s/k3s-test-1-worker-3/default.nix

@@ -0,0 +1,30 @@
+{
+  config,
+  pkgs,
+  myvars,
+  mylib,
+  ...
+}:
+let
+  hostName = "k3s-test-1-worker-3"; # define your hostname.
+
+  coreModule = mylib.genKubeVirtGuestModule {
+    inherit pkgs hostName;
+    inherit (myvars) networking;
+  };
+  k3sModule = mylib.genK3sAgentModule {
+    inherit pkgs;
+    tokenFile = config.age.secrets."k3s-test-1-token".path;
+    # use my own domain & kube-vip's virtual IP for the API server
+    # so that the API server can always be accessed even if some nodes are down
+    masterHost = "test-cluster-1.writefor.fun";
+    # workloads run on the workers; the masters are tainted (see k3s-test-1-master-*)
+    nodeLabels = [ "node-role.kubernetes.io/worker=true" ];
+  };
+in
+{
+  imports = (mylib.scanPaths ./.) ++ [
+    coreModule
+    k3sModule
+  ];
+}

outputs/x86_64-linux/src/k3s-test-1-worker-1.nix

@@ -0,0 +1,41 @@
+{
+  # NOTE: the args not used in this file CAN NOT be removed!
+  # because haumea pass argument lazily,
+  # and these arguments are used in the functions like `mylib.nixosSystem`, `mylib.colmenaSystem`, etc.
+  inputs,
+  lib,
+  mylib,
+  myvars,
+  system,
+  genSpecialArgs,
+  ...
+}@args:
+let
+  name = "k3s-test-1-worker-1";
+  tags = [ name ];
+  ssh-user = "root";
+
+  modules = {
+    nixos-modules =
+      (map mylib.relativeToRoot [
+        # common
+        "secrets/nixos.nix"
+        "modules/nixos/server/server.nix"
+        "modules/nixos/server/kubevirt-hardware-configuration.nix"
+        # host specific
+        "hosts/k8s/${name}"
+      ])
+      ++ [
+        { modules.secrets.server.kubernetes.enable = true; }
+      ];
+  };
+
+  systemArgs = modules // args;
+in
+{
+  nixosConfigurations.${name} = mylib.nixosSystem systemArgs;
+
+  colmena.${name} = mylib.colmenaSystem (systemArgs // { inherit tags ssh-user; });
+
+  packages.${name} = inputs.self.nixosConfigurations.${name}.config.system.build.images.kubevirt;
+}

outputs/x86_64-linux/src/k3s-test-1-worker-2.nix

@@ -0,0 +1,41 @@
+{
+  # NOTE: the args not used in this file CAN NOT be removed!
+  # because haumea pass argument lazily,
+  # and these arguments are used in the functions like `mylib.nixosSystem`, `mylib.colmenaSystem`, etc.
+  inputs,
+  lib,
+  mylib,
+  myvars,
+  system,
+  genSpecialArgs,
+  ...
+}@args:
+let
+  name = "k3s-test-1-worker-2";
+  tags = [ name ];
+  ssh-user = "root";
+
+  modules = {
+    nixos-modules =
+      (map mylib.relativeToRoot [
+        # common
+        "secrets/nixos.nix"
+        "modules/nixos/server/server.nix"
+        "modules/nixos/server/kubevirt-hardware-configuration.nix"
+        # host specific
+        "hosts/k8s/${name}"
+      ])
+      ++ [
+        { modules.secrets.server.kubernetes.enable = true; }
+      ];
+  };
+
+  systemArgs = modules // args;
+in
+{
+  nixosConfigurations.${name} = mylib.nixosSystem systemArgs;
+
+  colmena.${name} = mylib.colmenaSystem (systemArgs // { inherit tags ssh-user; });
+
+  packages.${name} = inputs.self.nixosConfigurations.${name}.config.system.build.images.kubevirt;
+}

outputs/x86_64-linux/src/k3s-test-1-worker-3.nix

@@ -0,0 +1,41 @@
+{
+  # NOTE: the args not used in this file CAN NOT be removed!
+  # because haumea pass argument lazily,
+  # and these arguments are used in the functions like `mylib.nixosSystem`, `mylib.colmenaSystem`, etc.
+  inputs,
+  lib,
+  mylib,
+  myvars,
+  system,
+  genSpecialArgs,
+  ...
+}@args:
+let
+  name = "k3s-test-1-worker-3";
+  tags = [ name ];
+  ssh-user = "root";
+
+  modules = {
+    nixos-modules =
+      (map mylib.relativeToRoot [
+        # common
+        "secrets/nixos.nix"
+        "modules/nixos/server/server.nix"
+        "modules/nixos/server/kubevirt-hardware-configuration.nix"
+        # host specific
+        "hosts/k8s/${name}"
+      ])
+      ++ [
+        { modules.secrets.server.kubernetes.enable = true; }
+      ];
+  };
+
+  systemArgs = modules // args;
+in
+{
+  nixosConfigurations.${name} = mylib.nixosSystem systemArgs;
+
+  colmena.${name} = mylib.colmenaSystem (systemArgs // { inherit tags ssh-user; });
+
+  packages.${name} = inputs.self.nixosConfigurations.${name}.config.system.build.images.kubevirt;
+}

vars/networking.nix

@@ -119,6 +119,21 @@ rec {
       iface = "enp2s0";
       ipv4 = "192.168.5.116";
     };
+    k3s-test-1-worker-1 = {
+      # KubeVirt VM
+      iface = "enp2s0";
+      ipv4 = "192.168.5.111";
+    };
+    k3s-test-1-worker-2 = {
+      # KubeVirt VM
+      iface = "enp2s0";
+      ipv4 = "192.168.5.112";
+    };
+    k3s-test-1-worker-3 = {
+      # KubeVirt VM
+      iface = "enp2s0";
+      ipv4 = "192.168.5.113";
+    };
   };
 
   hostsInterface = lib.attrsets.mapAttrs (key: val: {