feat(k8s): add the k3s-test-1 worker nodes (#307)

Add worker-1 (4 vCPU / 16 GiB) on shoryu, and worker-2 (4 vCPU / 16 GiB) plus worker-3 (2 vCPU / 8 GiB) on shushou, as microVMs. They come with a genK3sAgentModule and the per-host/output/home-manager wiring, and are labelled node-role.kubernetes.io/worker=true. The masters keep their control-plane:NoSchedule taint, so workloads land on the workers.

home/hosts/linux/k3s-test-1-master-2.nix

@@ -0,0 +1,3 @@
+{
+  imports = [ ../../linux/core.nix ];
+}

home/hosts/linux/k3s-test-1-master-3.nix

@@ -0,0 +1,3 @@
+{
+  imports = [ ../../linux/core.nix ];
+}

home/hosts/linux/k3s-test-1-worker-1.nix

@@ -0,0 +1,3 @@
+{
+  imports = [ ../../linux/core.nix ];
+}

home/hosts/linux/k3s-test-1-worker-2.nix

@@ -0,0 +1,3 @@
+{
+  imports = [ ../../linux/core.nix ];
+}

home/hosts/linux/k3s-test-1-worker-3.nix

@@ -0,0 +1,3 @@
+{
+  imports = [ ../../linux/core.nix ];
+}

hosts/12kingdoms-shoryu/microvm.nix

@@ -24,8 +24,23 @@
     config.imports = [ ../k8s/k3s-test-1-master-1 ];
   };
 
+  microvm.vms.k3s-test-1-worker-1 = {
+    autostart = true;
+    restartIfChanged = true;
+    specialArgs = {
+      inherit
+        myvars
+        mylib
+        agenix
+        mysecrets
+        ;
+    };
+    config.imports = [ ../k8s/k3s-test-1-worker-1 ];
+  };
+
   # Attach the guest's tap to the VM bridge, the same way the physical NIC is
-  # attached. The tap name is derived from the guest IP (192.168.5.114 -> vm114).
+  # attached. The tap name is derived from the guest IP (192.168.5.114 -> vm114,
+  # 192.168.5.111 -> vm111), as IFNAMSIZ caps interface names at 15 characters.
   systemd.network.networks."20-vm114" = {
     matchConfig.Name = [ "vm114" ];
     networkConfig = {
@@ -34,4 +49,12 @@
     };
     linkConfig.RequiredForOnline = "no";
   };
+  systemd.network.networks."20-vm111" = {
+    matchConfig.Name = [ "vm111" ];
+    networkConfig = {
+      LinkLocalAddressing = "no";
+      Bridge = "br0";
+    };
+    linkConfig.RequiredForOnline = "no";
+  };
 }

hosts/12kingdoms-shushou/microvm.nix

@@ -24,8 +24,38 @@
     config.imports = [ ../k8s/k3s-test-1-master-3 ];
   };
 
+  microvm.vms.k3s-test-1-worker-2 = {
+    autostart = true;
+    restartIfChanged = true;
+    specialArgs = {
+      inherit
+        myvars
+        mylib
+        agenix
+        mysecrets
+        ;
+    };
+    config.imports = [ ../k8s/k3s-test-1-worker-2 ];
+  };
+
+  microvm.vms.k3s-test-1-worker-3 = {
+    autostart = true;
+    restartIfChanged = true;
+    specialArgs = {
+      inherit
+        myvars
+        mylib
+        agenix
+        mysecrets
+        ;
+    };
+    config.imports = [ ../k8s/k3s-test-1-worker-3 ];
+  };
+
   # Attach the guest's tap to the VM bridge, the same way the physical NIC is
-  # attached. The tap name is derived from the guest IP (192.168.5.116 -> vm116).
+  # attached. The tap name is derived from the guest IP (192.168.5.116 -> vm116,
+  # 192.168.5.112 -> vm112, 192.168.5.113 -> vm113), as IFNAMSIZ caps interface
+  # names at 15 characters.
   systemd.network.networks."20-vm116" = {
     matchConfig.Name = [ "vm116" ];
     networkConfig = {
@@ -34,4 +64,20 @@
     };
     linkConfig.RequiredForOnline = "no";
   };
+  systemd.network.networks."20-vm112" = {
+    matchConfig.Name = [ "vm112" ];
+    networkConfig = {
+      LinkLocalAddressing = "no";
+      Bridge = "br0";
+    };
+    linkConfig.RequiredForOnline = "no";
+  };
+  systemd.network.networks."20-vm113" = {
+    matchConfig.Name = [ "vm113" ];
+    networkConfig = {
+      LinkLocalAddressing = "no";
+      Bridge = "br0";
+    };
+    linkConfig.RequiredForOnline = "no";
+  };
 }

hosts/README.md

@@ -46,7 +46,8 @@ Named after "Twelve Kingdoms":
 #### `k8s` - Kubernetes Infrastructure
 
 - **VM Cluster**: 3 physical mini PCs (shoryu, shushou, youko) running all VMs
-- **K3s Testing**: `k3s-test-1-master-{1,2,3}`, running as microVMs on those hosts
+- **K3s Testing**: `k3s-test-1-master-{1,2,3}` (control plane) + `k3s-test-1-worker-{1,2,3}`
+  (workloads; two on `shushou`, one on `shoryu`), running as microVMs
 
 ### External Systems
 

hosts/k8s/README.md

@@ -30,7 +30,14 @@ To move an existing VM over in place, copy the host key and `machine-id` from it
 `etc.img`, and `/var/lib/rancher/k3s/server` into `var.img`, **before the first boot** — then it
 keeps its identity and etcd membership instead of bootstrapping a new cluster.
 
-1. `k3s-test-1-master-{1,2,3}` — running as microVMs
+1. `k3s-test-1-master-{1,2,3}` — control plane, running as microVMs; tainted
+   `node-role.kubernetes.io/control-plane:NoSchedule`
+1. `k3s-test-1-worker-{1,2,3}` — workloads, running as microVMs
+   (`node-role.kubernetes.io/worker=true`)
+
+Placement: `worker-1` (4 vCPU / 16 GiB) runs on `shoryu`; `worker-2` (4 vCPU / 16 GiB) and
+`worker-3` (2 vCPU / 8 GiB) run on `shushou`. `youko` has no worker because it has the least free
+memory and also runs the homelab services.
 
 ## TODO / Known issues
 

hosts/k8s/k3s-test-1-worker-1/default.nix

@@ -0,0 +1,48 @@
+{
+  config,
+  pkgs,
+  lib,
+  myvars,
+  mylib,
+  ...
+}:
+let
+  hostName = "k3s-test-1-worker-1"; # Define your hostname.
+
+  coreModule = mylib.genMicrovmGuestModule {
+    inherit pkgs hostName;
+    inherit (myvars) networking;
+    vcpu = 4;
+    mem = 16384;
+    varSize = 20480;
+  };
+  k3sModule = mylib.genK3sAgentModule {
+    inherit pkgs;
+    tokenFile = config.age.secrets."k3s-test-1-token".path;
+    # use my own domain & kube-vip's virtual IP for the API server
+    # so that the API server can always be accessed even if some nodes are down
+    masterHost = "test-cluster-1.writefor.fun";
+    # workloads run on the workers; the masters are tainted (see k3s-test-1-master-*)
+    nodeLabels = [ "node-role.kubernetes.io/worker=true" ];
+  };
+in
+{
+  imports =
+    (mylib.scanPaths ./.)
+    ++ (map mylib.relativeToRoot [
+      "secrets/nixos.nix"
+      "modules/nixos/server/server.nix"
+    ])
+    ++ [
+      coreModule
+      k3sModule
+    ];
+
+  # k3s-test-1-token lives in this secret group
+  modules.secrets.server.kubernetes.enable = true;
+
+  # The guest reuses the host's already-instantiated nixpkgs (shared store),
+  # so nixpkgs.config may not be set. The shared base modules set
+  # nixpkgs.config.allowUnfree, so force it empty here.
+  nixpkgs.config = lib.mkForce { };
+}

hosts/k8s/k3s-test-1-worker-2/default.nix

@@ -0,0 +1,48 @@
+{
+  config,
+  pkgs,
+  lib,
+  myvars,
+  mylib,
+  ...
+}:
+let
+  hostName = "k3s-test-1-worker-2"; # Define your hostname.
+
+  coreModule = mylib.genMicrovmGuestModule {
+    inherit pkgs hostName;
+    inherit (myvars) networking;
+    vcpu = 4;
+    mem = 16384;
+    varSize = 20480;
+  };
+  k3sModule = mylib.genK3sAgentModule {
+    inherit pkgs;
+    tokenFile = config.age.secrets."k3s-test-1-token".path;
+    # use my own domain & kube-vip's virtual IP for the API server
+    # so that the API server can always be accessed even if some nodes are down
+    masterHost = "test-cluster-1.writefor.fun";
+    # workloads run on the workers; the masters are tainted (see k3s-test-1-master-*)
+    nodeLabels = [ "node-role.kubernetes.io/worker=true" ];
+  };
+in
+{
+  imports =
+    (mylib.scanPaths ./.)
+    ++ (map mylib.relativeToRoot [
+      "secrets/nixos.nix"
+      "modules/nixos/server/server.nix"
+    ])
+    ++ [
+      coreModule
+      k3sModule
+    ];
+
+  # k3s-test-1-token lives in this secret group
+  modules.secrets.server.kubernetes.enable = true;
+
+  # The guest reuses the host's already-instantiated nixpkgs (shared store),
+  # so nixpkgs.config may not be set. The shared base modules set
+  # nixpkgs.config.allowUnfree, so force it empty here.
+  nixpkgs.config = lib.mkForce { };
+}

hosts/k8s/k3s-test-1-worker-3/default.nix

@@ -0,0 +1,48 @@
+{
+  config,
+  pkgs,
+  lib,
+  myvars,
+  mylib,
+  ...
+}:
+let
+  hostName = "k3s-test-1-worker-3"; # Define your hostname.
+
+  coreModule = mylib.genMicrovmGuestModule {
+    inherit pkgs hostName;
+    inherit (myvars) networking;
+    vcpu = 2;
+    mem = 8192;
+    varSize = 20480;
+  };
+  k3sModule = mylib.genK3sAgentModule {
+    inherit pkgs;
+    tokenFile = config.age.secrets."k3s-test-1-token".path;
+    # use my own domain & kube-vip's virtual IP for the API server
+    # so that the API server can always be accessed even if some nodes are down
+    masterHost = "test-cluster-1.writefor.fun";
+    # workloads run on the workers; the masters are tainted (see k3s-test-1-master-*)
+    nodeLabels = [ "node-role.kubernetes.io/worker=true" ];
+  };
+in
+{
+  imports =
+    (mylib.scanPaths ./.)
+    ++ (map mylib.relativeToRoot [
+      "secrets/nixos.nix"
+      "modules/nixos/server/server.nix"
+    ])
+    ++ [
+      coreModule
+      k3sModule
+    ];
+
+  # k3s-test-1-token lives in this secret group
+  modules.secrets.server.kubernetes.enable = true;
+
+  # The guest reuses the host's already-instantiated nixpkgs (shared store),
+  # so nixpkgs.config may not be set. The shared base modules set
+  # nixpkgs.config.allowUnfree, so force it empty here.
+  nixpkgs.config = lib.mkForce { };
+}

lib/default.nix

@@ -7,6 +7,7 @@
   attrs = import ./attrs.nix { inherit lib; };
 
   genK3sServerModule = import ./genK3sServerModule.nix;
+  genK3sAgentModule = import ./genK3sAgentModule.nix;
   genVmHostModule = import ./genVmHostModule.nix;
   genMicrovmGuestModule = import ./genMicrovmGuestModule.nix;
 

lib/genK3sAgentModule.nix

@@ -0,0 +1,45 @@
+{
+  pkgs,
+  masterHost,
+  tokenFile,
+  nodeLabels ? [ ],
+  k3sExtraArgs ? [ ],
+  ...
+}:
+let
+  package = pkgs.k3s;
+in
+{
+  environment.systemPackages = [ package ];
+
+  # Kernel modules required by cilium
+  boot.kernelModules = [
+    "ip6_tables"
+    "ip6table_mangle"
+    "ip6table_raw"
+    "ip6table_filter"
+  ];
+  networking.enableIPv6 = true;
+  networking.nat = {
+    enable = true;
+    enableIPv6 = true;
+  };
+
+  services.k3s = {
+    enable = true;
+    inherit package tokenFile;
+
+    role = "agent";
+    serverAddr = "https://${masterHost}:6443";
+    # https://docs.k3s.io/cli/agent
+    extraFlags =
+      let
+        flagList = [
+          "--data-dir /var/lib/rancher/k3s"
+        ]
+        ++ (map (label: "--node-label=${label}") nodeLabels)
+        ++ k3sExtraArgs;
+      in
+      pkgs.lib.concatStringsSep " " flagList;
+  };
+}

outputs/x86_64-linux/src/k3s-test-1-worker-1.nix

@@ -0,0 +1,40 @@
+{
+  # NOTE: the args not used in this file CAN NOT be removed!
+  # because haumea pass argument lazily,
+  # and these arguments are used in the functions like `mylib.nixosSystem`, `mylib.colmenaSystem`, etc.
+  inputs,
+  lib,
+  mylib,
+  myvars,
+  system,
+  genSpecialArgs,
+  ...
+}@args:
+let
+  name = "k3s-test-1-worker-1";
+  tags = [ name ];
+  ssh-user = "root";
+
+  modules = {
+    # the host dir is the complete config (it pulls in the shared modules)
+    nixos-modules =
+      (map mylib.relativeToRoot [
+        "hosts/k8s/${name}"
+      ])
+      ++ [
+        # this runs as a MicroVM
+        inputs.microvm.nixosModules.microvm
+        { modules.secrets.server.kubernetes.enable = true; }
+      ];
+    home-modules = map mylib.relativeToRoot [
+      "home/hosts/linux/${name}.nix"
+    ];
+  };
+
+  systemArgs = modules // args;
+in
+{
+  nixosConfigurations.${name} = mylib.nixosSystem systemArgs;
+
+  colmena.${name} = mylib.colmenaSystem (systemArgs // { inherit tags ssh-user; });
+}

outputs/x86_64-linux/src/k3s-test-1-worker-2.nix

@@ -0,0 +1,40 @@
+{
+  # NOTE: the args not used in this file CAN NOT be removed!
+  # because haumea pass argument lazily,
+  # and these arguments are used in the functions like `mylib.nixosSystem`, `mylib.colmenaSystem`, etc.
+  inputs,
+  lib,
+  mylib,
+  myvars,
+  system,
+  genSpecialArgs,
+  ...
+}@args:
+let
+  name = "k3s-test-1-worker-2";
+  tags = [ name ];
+  ssh-user = "root";
+
+  modules = {
+    # the host dir is the complete config (it pulls in the shared modules)
+    nixos-modules =
+      (map mylib.relativeToRoot [
+        "hosts/k8s/${name}"
+      ])
+      ++ [
+        # this runs as a MicroVM
+        inputs.microvm.nixosModules.microvm
+        { modules.secrets.server.kubernetes.enable = true; }
+      ];
+    home-modules = map mylib.relativeToRoot [
+      "home/hosts/linux/${name}.nix"
+    ];
+  };
+
+  systemArgs = modules // args;
+in
+{
+  nixosConfigurations.${name} = mylib.nixosSystem systemArgs;
+
+  colmena.${name} = mylib.colmenaSystem (systemArgs // { inherit tags ssh-user; });
+}

outputs/x86_64-linux/src/k3s-test-1-worker-3.nix

@@ -0,0 +1,40 @@
+{
+  # NOTE: the args not used in this file CAN NOT be removed!
+  # because haumea pass argument lazily,
+  # and these arguments are used in the functions like `mylib.nixosSystem`, `mylib.colmenaSystem`, etc.
+  inputs,
+  lib,
+  mylib,
+  myvars,
+  system,
+  genSpecialArgs,
+  ...
+}@args:
+let
+  name = "k3s-test-1-worker-3";
+  tags = [ name ];
+  ssh-user = "root";
+
+  modules = {
+    # the host dir is the complete config (it pulls in the shared modules)
+    nixos-modules =
+      (map mylib.relativeToRoot [
+        "hosts/k8s/${name}"
+      ])
+      ++ [
+        # this runs as a MicroVM
+        inputs.microvm.nixosModules.microvm
+        { modules.secrets.server.kubernetes.enable = true; }
+      ];
+    home-modules = map mylib.relativeToRoot [
+      "home/hosts/linux/${name}.nix"
+    ];
+  };
+
+  systemArgs = modules // args;
+in
+{
+  nixosConfigurations.${name} = mylib.nixosSystem systemArgs;
+
+  colmena.${name} = mylib.colmenaSystem (systemArgs // { inherit tags ssh-user; });
+}