feat: new k3s cluster

hosts/k8s/README.md

@@ -0,0 +1,25 @@
+# Kubernetes Clusters
+
+> WIP, not finished yet.
+
+I'm running two Kubernetes clusters, one for production and one for testing.
+
+I prefer to use [k3s] as the Kubernetes distribution, because it's lightweight, easy to install, and full featured(see [what-have-k3s-removed-from-upstream-kubernetes] for details).
+
+## Hosts
+
+1. For production:
+   1. `k3s-prod-1-master-1`
+   1. `k3s-prod-1-master-2`
+   1. `k3s-prod-1-master-3`
+   2. `k3s-prod-1-worker-1`
+   2. `k3s-prod-1-worker-2`
+   2. `k3s-prod-1-worker-3`
+1. For testing:. 
+   1. `k3s-test-1-master-1`
+   2. `k3s-test-1-worker-1`
+   3. `k3s-test-1-worker-2`
+   4. `k3s-test-1-worker-3`
+
+[k3s]: https://github.com/k3s-io/k3s/
+[what-have-k3s-removed-from-upstream-kubernetes]: https://github.com/k3s-io/k3s/?tab=readme-ov-file#what-have-you-removed-from-upstream-kubernetes

hosts/k8s/k3s_prod_1_master_1/default.nix

@@ -0,0 +1,17 @@
+{
+  vars_networking,
+  mylib,
+  ...
+}: let
+  hostName = "k3s-prod-1-master-1"; # Define your hostname.
+  k8sLib = import ../lib.nix;
+  coreModule = k8sLib.genCoreModule {
+    inherit hostName vars_networking;
+  };
+in {
+  imports =
+    (mylib.scanPaths ./.)
+    ++ [
+      coreModule
+    ];
+}

hosts/k8s/k3s_prod_1_master_1/k3s.nix

@@ -0,0 +1,38 @@
+{
+  config,
+  pkgs,
+  ...
+}: let
+  package = pkgs.k3s_1_29;
+in {
+  environment.systemPackages = with pkgs; [
+    package
+    k9s
+    kubectl
+    istioctl
+    kubernetes-helm
+
+    skopeo
+    dive # explore docker layers
+  ];
+  services.k3s = {
+    inherit package;
+    enable = true;
+
+    # Initialize HA cluster using an embedded etcd datastore.
+    # If you are configuring an HA cluster with an embedded etcd,
+    # the 1st server must have `clusterInit = true`
+    # and other servers must connect to it using serverAddr.
+    clusterInit = true;
+    role = "server";
+    tokenFile = config.age.secrets."k3s-prod-1-token".path;
+    # https://docs.k3s.io/cli/server
+    extraFlags =
+      " --write-kubeconfig /etc/k3s/kubeconfig.yml"
+      + " --write-kubeconfig-mode 644"
+      + " --service-node-port-range 80-32767"
+      + " --data-dir /var/lib/rancher/k3s"
+      + " --etcd-expose-metrics true"
+      + '' --etcd-snapshot-schedule-cron "0 */12 * * *"'';
+  };
+}

hosts/k8s/k3s_prod_1_master_2/default.nix

@@ -0,0 +1,17 @@
+{
+  vars_networking,
+  mylib,
+  ...
+}: let
+  hostName = "k3s-prod-1-master-2"; # Define your hostname.
+  k8sLib = import ../lib.nix;
+  coreModule = k8sLib.genCoreModule {
+    inherit hostName vars_networking;
+  };
+in {
+  imports =
+    (mylib.scanPaths ./.)
+    ++ [
+      coreModule
+    ];
+}

hosts/k8s/k3s_prod_1_master_2/k3s.nix

@@ -0,0 +1,27 @@
+{
+  config,
+  pkgs,
+  vars_networking,
+  ...
+}: let
+  serverName = "k3s-prod-1-master-1";
+  serverIp = vars_networking.hostAddress.${serverName}.address;
+  package = pkgs.k3s_1_29;
+in {
+  environment.systemPackages = [package];
+  services.k3s = {
+    inherit package;
+    enable = true;
+    role = "server";
+    serverAddr = "https://${serverIp}:6443";
+    tokenFile = config.age.secrets."k3s-prod-1-token".path;
+    # https://docs.k3s.io/cli/server
+    extraFlags =
+      " --write-kubeconfig /etc/k3s/kubeconfig.yml"
+      + " --write-kubeconfig-mode 644"
+      + " --service-node-port-range 80-32767"
+      + " --data-dir /var/lib/rancher/k3s"
+      + " --etcd-expose-metrics true"
+      + '' --etcd-snapshot-schedule-cron "0 */12 * * *"'';
+  };
+}

hosts/k8s/k3s_prod_1_master_3/default.nix

@@ -0,0 +1,17 @@
+{
+  vars_networking,
+  mylib,
+  ...
+}: let
+  hostName = "k3s-prod-1-master-3"; # Define your hostname.
+  k8sLib = import ../lib.nix;
+  coreModule = k8sLib.genCoreModule {
+    inherit hostName vars_networking;
+  };
+in {
+  imports =
+    (mylib.scanPaths ./.)
+    ++ [
+      coreModule
+    ];
+}

hosts/k8s/k3s_prod_1_master_3/k3s.nix

@@ -0,0 +1,27 @@
+{
+  config,
+  pkgs,
+  vars_networking,
+  ...
+}: let
+  serverName = "k3s-prod-1-master-1";
+  serverIp = vars_networking.hostAddress.${serverName}.address;
+  package = pkgs.k3s_1_29;
+in {
+  environment.systemPackages = [package];
+  services.k3s = {
+    inherit package;
+    enable = true;
+    role = "server";
+    serverAddr = "https://${serverIp}:6443";
+    tokenFile = config.age.secrets."k3s-prod-1-token".path;
+    # https://docs.k3s.io/cli/server
+    extraFlags =
+      " --write-kubeconfig /etc/k3s/kubeconfig.yml"
+      + " --write-kubeconfig-mode 644"
+      + " --service-node-port-range 80-32767"
+      + " --data-dir /var/lib/rancher/k3s"
+      + " --etcd-expose-metrics true"
+      + '' --etcd-snapshot-schedule-cron "0 */12 * * *"'';
+  };
+}

hosts/k8s/k3s_prod_1_worker_1/default.nix

@@ -0,0 +1,17 @@
+{
+  vars_networking,
+  mylib,
+  ...
+}: let
+  hostName = "k3s-prod-1-worker-1"; # Define your hostname.
+  k8sLib = import ../lib.nix;
+  coreModule = k8sLib.genCoreModule {
+    inherit hostName vars_networking;
+  };
+in {
+  imports =
+    (mylib.scanPaths ./.)
+    ++ [
+      coreModule
+    ];
+}

hosts/k8s/k3s_prod_1_worker_1/k3s.nix

@@ -0,0 +1,21 @@
+{
+  config,
+  pkgs,
+  vars_networking,
+  ...
+}: let
+  serverName = "k3s-prod-1-master-1";
+  serverIp = vars_networking.hostAddress.${serverName}.address;
+  package = pkgs.k3s_1_29;
+in {
+  environment.systemPackages = [package];
+  services.k3s = {
+    inherit package;
+    enable = true;
+    role = "agent";
+    serverAddr = "https://${serverIp}:6443";
+    tokenFile = config.age.secrets."k3s-prod-1-token".path;
+    # https://docs.k3s.io/cli/agent
+    extraFlags = "--data-dir /var/lib/rancher/k3s";
+  };
+}

hosts/k8s/k3s_prod_1_worker_2/default.nix

@@ -0,0 +1,17 @@
+{
+  vars_networking,
+  mylib,
+  ...
+}: let
+  hostName = "k3s-prod-1-worker-2"; # Define your hostname.
+  k8sLib = import ../lib.nix;
+  coreModule = k8sLib.genCoreModule {
+    inherit hostName vars_networking;
+  };
+in {
+  imports =
+    (mylib.scanPaths ./.)
+    ++ [
+      coreModule
+    ];
+}

hosts/k8s/k3s_prod_1_worker_2/k3s.nix

@@ -0,0 +1,21 @@
+{
+  config,
+  pkgs,
+  vars_networking,
+  ...
+}: let
+  serverName = "k3s-prod-1-master-1";
+  serverIp = vars_networking.hostAddress.${serverName}.address;
+  package = pkgs.k3s_1_29;
+in {
+  environment.systemPackages = [package];
+  services.k3s = {
+    inherit package;
+    enable = true;
+    role = "agent";
+    serverAddr = "https://${serverIp}:6443";
+    tokenFile = config.age.secrets."k3s-prod-1-token".path;
+    # https://docs.k3s.io/cli/agent
+    extraFlags = "--data-dir /var/lib/rancher/k3s";
+  };
+}

hosts/k8s/k3s_prod_1_worker_3/default.nix

@@ -0,0 +1,17 @@
+{
+  vars_networking,
+  mylib,
+  ...
+}: let
+  hostName = "k3s-prod-1-worker-3"; # Define your hostname.
+  k8sLib = import ../lib.nix;
+  coreModule = k8sLib.genCoreModule {
+    inherit hostName vars_networking;
+  };
+in {
+  imports =
+    (mylib.scanPaths ./.)
+    ++ [
+      coreModule
+    ];
+}

hosts/k8s/k3s_prod_1_worker_3/k3s.nix

@@ -0,0 +1,21 @@
+{
+  config,
+  pkgs,
+  vars_networking,
+  ...
+}: let
+  serverName = "k3s-prod-1-master-1";
+  serverIp = vars_networking.hostAddress.${serverName}.address;
+  package = pkgs.k3s_1_29;
+in {
+  environment.systemPackages = [package];
+  services.k3s = {
+    inherit package;
+    enable = true;
+    role = "agent";
+    serverAddr = "https://${serverIp}:6443";
+    tokenFile = config.age.secrets."k3s-prod-1-token".path;
+    # https://docs.k3s.io/cli/agent
+    extraFlags = "--data-dir /var/lib/rancher/k3s";
+  };
+}

hosts/k8s/lib.nix

@@ -0,0 +1,43 @@
+{
+  genCoreModule = {
+    hostName,
+    vars_networking,
+  }: let
+    hostAddress = vars_networking.hostAddress.${hostName};
+  in {
+    # supported file systems, so we can mount any removable disks with these filesystems
+    boot.supportedFilesystems = [
+      "ext4"
+      "btrfs"
+      "xfs"
+      #"zfs"
+      "ntfs"
+      "fat"
+      "vfat"
+      "exfat"
+      "cifs" # mount windows share
+    ];
+
+    boot.kernelModules = ["kvm-amd"];
+    boot.extraModprobeConfig = "options kvm_amd nested=1"; # for amd cpu
+
+    networking = {
+      inherit hostName;
+      inherit (vars_networking) defaultGateway nameservers;
+
+      networkmanager.enable = false;
+      interfaces.ens18 = {
+        useDHCP = false;
+        ipv4.addresses = [hostAddress];
+      };
+    };
+
+    # This value determines the NixOS release from which the default
+    # settings for stateful data, like file locations and database versions
+    # on your system were taken. It‘s perfectly fine and recommended to leave
+    # this value at the release version of the first install of this system.
+    # Before changing this value read the documentation for this option
+    # (e.g. man configuration.nix or on https://nixos.org/nixos/options.html).
+    system.stateVersion = "23.11"; # Did you read the comment?
+  };
+}