From 84c21ccd34448c1db33cf013a09f96081f8e95e2 Mon Sep 17 00:00:00 2001 From: Ryan Yin Date: Sat, 23 Mar 2024 01:42:17 +0800 Subject: [PATCH] refactor(vars,hosts): networking & k3s clusters --- Justfile | 39 ----- hosts/12kingdoms-rakushun/default.nix | 14 +- .../tailscale.nix | 0 hosts/12kingdoms-suzu/default.nix | 7 +- hosts/README.md | 3 - hosts/homelab-tailscale-gw/default.nix | 48 ------ hosts/idols-ai/default.nix | 9 +- hosts/idols-kana/default.nix | 7 +- hosts/idols-ruby/default.nix | 7 +- hosts/idols-ruby/prometheus/default.nix | 10 +- hosts/k8s/k3s-prod-1-master-1/default.nix | 13 +- hosts/k8s/k3s-prod-1-master-2/default.nix | 13 +- hosts/k8s/k3s-prod-1-master-2/k3s.nix | 31 ---- hosts/k8s/k3s-prod-1-master-3/default.nix | 13 +- hosts/k8s/k3s-prod-1-master-3/k3s.nix | 31 ---- hosts/k8s/k3s-prod-1-worker-1/default.nix | 12 +- hosts/k8s/k3s-prod-1-worker-2/default.nix | 12 +- hosts/k8s/k3s-prod-1-worker-2/k3s.nix | 23 --- hosts/k8s/k3s-prod-1-worker-3/default.nix | 12 +- hosts/k8s/k3s-prod-1-worker-3/k3s.nix | 23 --- hosts/k8s/kubevirt-shoryu/default.nix | 15 +- hosts/k8s/kubevirt-shoryu/k3s.nix | 42 ------ hosts/k8s/kubevirt-shushou/default.nix | 15 +- hosts/k8s/kubevirt-shushou/k3s.nix | 42 ------ hosts/k8s/kubevirt-youko/default.nix | 15 +- hosts/k8s/kubevirt-youko/k3s.nix | 42 ------ hosts/k8s/lib.nix | 86 ----------- hosts/rolling-girls-nozomi/default.nix | 7 +- hosts/rolling-girls-yukina/default.nix | 7 +- lib/default.nix | 6 + .../k3s.nix => lib/genK3sAgentModule.nix | 10 +- .../k3s.nix => lib/genK3sServerModule.nix | 37 +++-- lib/genKubeVirtCoreModule.nix | 78 ++++++++++ outputs/README.md | 1 - .../x86_64-linux/src/homelab-tailscale-gw.nix | 37 ----- vars/networking.nix | 140 ++++++++++++++---- 36 files changed, 336 insertions(+), 571 deletions(-) rename hosts/{homelab-tailscale-gw => 12kingdoms-rakushun}/tailscale.nix (100%) delete mode 100644 hosts/homelab-tailscale-gw/default.nix delete mode 100644 hosts/k8s/k3s-prod-1-master-2/k3s.nix delete mode 100644 hosts/k8s/k3s-prod-1-master-3/k3s.nix delete mode 100644 hosts/k8s/k3s-prod-1-worker-2/k3s.nix delete mode 100644 hosts/k8s/k3s-prod-1-worker-3/k3s.nix delete mode 100644 hosts/k8s/kubevirt-shoryu/k3s.nix delete mode 100644 hosts/k8s/kubevirt-shushou/k3s.nix delete mode 100644 hosts/k8s/kubevirt-youko/k3s.nix delete mode 100644 hosts/k8s/lib.nix rename hosts/k8s/k3s-prod-1-worker-1/k3s.nix => lib/genK3sAgentModule.nix (62%) rename hosts/k8s/k3s-prod-1-master-1/k3s.nix => lib/genK3sServerModule.nix (52%) create mode 100644 lib/genKubeVirtCoreModule.nix delete mode 100644 outputs/x86_64-linux/src/homelab-tailscale-gw.nix diff --git a/Justfile b/Justfile index 2fbdc6dd..6a4deaa7 100644 --- a/Justfile +++ b/Justfile @@ -129,26 +129,6 @@ ruby-local mode="default": kana: colmena apply --on '@kana' --verbose --show-trace -tailscale: - colmena apply --on '@tailscale-gw' --verbose --show-trace - -# pve-aqua: -# nom build .#aquamarine -# rsync -avz --progress --copy-links result root@um560:/var/lib/vz/dump/vzdump-qemu-aquamarine.vma.zst -# -# pve-ruby: -# nom build .#ruby -# rsync -avz --progress --copy-links result root@um560:/var/lib/vz/dump/vzdump-qemu-ruby.vma.zst -# -# pve-kana: -# nom build .#kana -# rsync -avz --progress --copy-links result root@gtr5:/var/lib/vz/dump/vzdump-qemu-kana.vma.zst -# -# pve-tsgw: -# nom build .#tailscale-gw -# rsync -avz --progress --copy-links result root@um560:/var/lib/vz/dump/vzdump-qemu-tailscale-gw.vma.zst -# - ############################################################################ # # Kubernetes related commands @@ -164,25 +144,6 @@ master: worker: colmena apply --on '@k3s-prod-1-worker-*' --verbose --show-trace -# pve-k8s: -# nom build .#k3s-prod-1-master-1 -# rsync -avz --progress --copy-links result root@um560:/var/lib/vz/dump/vzdump-qemu-k3s-prod-1-master-1.vma.zst -# -# nom build .#k3s-prod-1-master-2 -# rsync -avz --progress --copy-links result root@gtr5:/var/lib/vz/dump/vzdump-qemu-k3s-prod-1-master-2.vma.zst -# -# nom build .#k3s-prod-1-master-3 -# rsync -avz --progress --copy-links result root@s500plus:/var/lib/vz/dump/vzdump-qemu-k3s-prod-1-master-3.vma.zst -# -# nom build .#k3s-prod-1-worker-1 -# rsync -avz --progress --copy-links result root@gtr5:/var/lib/vz/dump/vzdump-qemu-k3s-prod-1-worker-1.vma.zst -# -# nom build .#k3s-prod-1-worker-2 -# rsync -avz --progress --copy-links result root@s500plus:/var/lib/vz/dump/vzdump-qemu-k3s-prod-1-worker-2.vma.zst -# -# nom build .#k3s-prod-1-worker-3 -# rsync -avz --progress --copy-links result root@s500plus:/var/lib/vz/dump/vzdump-qemu-k3s-prod-1-worker-3.vma.zst -# ############################################################################ # diff --git a/hosts/12kingdoms-rakushun/default.nix b/hosts/12kingdoms-rakushun/default.nix index dce1c3b5..654e6d63 100644 --- a/hosts/12kingdoms-rakushun/default.nix +++ b/hosts/12kingdoms-rakushun/default.nix @@ -11,7 +11,6 @@ ############################################################# let hostName = "rakushun"; # Define your hostname. - hostAddress = myvars.networking.hostAddress.${hostName}; in { imports = [ # import the rk3588 module, which contains the configuration for bootloader/kernel/firmware @@ -23,23 +22,14 @@ in { ./gitea.nix ./caddy.nix + ./tailscale.nix ]; networking = { inherit hostName; inherit (myvars.networking) defaultGateway nameservers; - + inherit (myvars.networking.hostsInterface.${hostName}) interfaces; networkmanager.enable = false; - # RJ45 port 1 - interfaces.enP4p65s0 = { - useDHCP = false; - ipv4.addresses = [hostAddress]; - }; - # RJ45 port 2 - # interfaces.enP3p49s0 = { - # useDHCP = false; - # ipv4.addresses = [hostAddress]; - # }; }; # This value determines the NixOS release from which the default diff --git a/hosts/homelab-tailscale-gw/tailscale.nix b/hosts/12kingdoms-rakushun/tailscale.nix similarity index 100% rename from hosts/homelab-tailscale-gw/tailscale.nix rename to hosts/12kingdoms-rakushun/tailscale.nix diff --git a/hosts/12kingdoms-suzu/default.nix b/hosts/12kingdoms-suzu/default.nix index 0e0f01cb..618d4dc7 100644 --- a/hosts/12kingdoms-suzu/default.nix +++ b/hosts/12kingdoms-suzu/default.nix @@ -11,7 +11,6 @@ ############################################################# let hostName = "suzu"; # Define your hostname. - hostAddress = myvars.networking.hostAddress.${hostName}; in { imports = [ # import the rk3588 module, which contains the configuration for bootloader/kernel/firmware @@ -25,12 +24,8 @@ in { networking = { inherit hostName; inherit (myvars.networking) defaultGateway nameservers; - + inherit (myvars.networking.hostsInterface.${hostName}) interfaces; networkmanager.enable = false; - interfaces.end1 = { - useDHCP = false; - ipv4.addresses = [hostAddress]; - }; }; # This value determines the NixOS release from which the default diff --git a/hosts/README.md b/hosts/README.md index b338fae5..3ffa9e47 100644 --- a/hosts/README.md +++ b/hosts/README.md @@ -11,9 +11,6 @@ prometheus, grafana, restic, etc. 4. `kana`: Yet another NixOS VM running some common applications, such as hompage, file browser, torrent downloader, etc. -3. Homelab: - 1. `tailscale-gw`: A tailscale subnet router(gateway) for accessing my homelab remotely. NixOS VM - running on Proxmox. 4. `rolling_girls`: My RISCV64 hosts. 1. `nozomi`: Lichee Pi 4A, TH1520(4xC910@2.0G), 16GB RAM + 32G eMMC + 128G SD Card. 2. `yukina`: Milk-V Mars, JH7110(4xU74@1.5 GHz), 4G RAM + No eMMC + 64G SD Card. diff --git a/hosts/homelab-tailscale-gw/default.nix b/hosts/homelab-tailscale-gw/default.nix deleted file mode 100644 index 03d3bee8..00000000 --- a/hosts/homelab-tailscale-gw/default.nix +++ /dev/null @@ -1,48 +0,0 @@ -{ - myvars, - mylib, - ... -}: -############################################################# -# -# Tailscale Gateway(homelab subnet router) - a NixOS VM running on Proxmox -# -############################################################# -let - hostName = "tailscale-gw"; # Define your hostname. - hostAddress = myvars.networking.hostAddress.${hostName}; -in { - imports = mylib.scanPaths ./.; - - # supported file systems, so we can mount any removable disks with these filesystems - boot.supportedFilesystems = [ - "ext4" - "btrfs" - "xfs" - "fat" - "vfat" - "exfat" - ]; - - networking = { - inherit hostName; - inherit (myvars.networking) nameservers; - - # Use mainGateway instead of defaultGateway to make NAT Traversal work - defaultGateway = myvars.networking.mainGateway; - - networkmanager.enable = false; - interfaces.ens18 = { - useDHCP = false; - ipv4.addresses = [hostAddress]; - }; - }; - - # This value determines the NixOS release from which the default - # settings for stateful data, like file locations and database versions - # on your system were taken. It‘s perfectly fine and recommended to leave - # this value at the release version of the first install of this system. - # Before changing this value read the documentation for this option - # (e.g. man configuration.nix or on https://nixos.org/nixos/options.html). - system.stateVersion = "23.11"; # Did you read the comment? -} diff --git a/hosts/idols-ai/default.nix b/hosts/idols-ai/default.nix index 42c9c7d4..339470d2 100644 --- a/hosts/idols-ai/default.nix +++ b/hosts/idols-ai/default.nix @@ -6,7 +6,6 @@ ############################################################# let hostName = "ai"; # Define your hostname. - hostAddress = myvars.networking.hostAddress.${hostName}; in { imports = [ ./cifs-mount.nix @@ -20,14 +19,8 @@ in { networking = { inherit hostName; inherit (myvars.networking) defaultGateway nameservers; - - wireless.enable = false; # Enables wireless support via wpa_supplicant. - # configures the network interface(include wireless) via `nmcli` & `nmtui` + inherit (myvars.networking.hostsInterface.${hostName}) interfaces; networkmanager.enable = false; - interfaces.enp5s0 = { - useDHCP = false; - ipv4.addresses = [hostAddress]; - }; }; # conflict with feature: containerd-snapshotter diff --git a/hosts/idols-kana/default.nix b/hosts/idols-kana/default.nix index b2e8a7d9..9a16acb0 100644 --- a/hosts/idols-kana/default.nix +++ b/hosts/idols-kana/default.nix @@ -10,7 +10,6 @@ ############################################################# let hostName = "kana"; # Define your hostname. - hostAddress = myvars.networking.hostAddress.${hostName}; in { imports = mylib.scanPaths ./.; @@ -33,12 +32,8 @@ in { networking = { inherit hostName; inherit (myvars.networking) defaultGateway nameservers; - + inherit (myvars.networking.hostsInterface.${hostName}) interfaces; networkmanager.enable = false; - interfaces.ens18 = { - useDHCP = false; - ipv4.addresses = [hostAddress]; - }; }; # This value determines the NixOS release from which the default diff --git a/hosts/idols-ruby/default.nix b/hosts/idols-ruby/default.nix index 83dbc656..bb107586 100644 --- a/hosts/idols-ruby/default.nix +++ b/hosts/idols-ruby/default.nix @@ -10,7 +10,6 @@ ############################################################# let hostName = "ruby"; # Define your hostname. - hostAddress = myvars.networking.hostAddress.${hostName}; in { imports = mylib.scanPaths ./.; @@ -35,12 +34,8 @@ in { networking = { inherit hostName; inherit (myvars.networking) defaultGateway nameservers; - + inherit (myvars.networking.hostsInterface.${hostName}) interfaces; networkmanager.enable = false; - interfaces.ens18 = { - useDHCP = false; - ipv4.addresses = [hostAddress]; - }; }; # This value determines the NixOS release from which the default diff --git a/hosts/idols-ruby/prometheus/default.nix b/hosts/idols-ruby/prometheus/default.nix index 4409c91b..ccc37bff 100644 --- a/hosts/idols-ruby/prometheus/default.nix +++ b/hosts/idols-ruby/prometheus/default.nix @@ -55,8 +55,8 @@ { # All my NixOS hosts. targets = - map (host: "${host.address}:9100") - (builtins.attrValues myvars.networking.hostAddress); + map (addr: "${addr.ipv4}:9100") + (builtins.attrValues myvars.networking.hostsAddr); labels.type = "node"; } ]; @@ -70,7 +70,7 @@ metrics_path = "/metrics"; static_configs = [ { - targets = ["${myvars.networking.hostAddress.aquamarine.address}:9153"]; + targets = ["${myvars.networking.hostsAddr.aquamarine.ipv4}:9153"]; labels.type = "app"; labels.app = "dnsmasq"; } @@ -83,7 +83,7 @@ metrics_path = "/metrics"; static_configs = [ { - targets = ["${myvars.networking.hostAddress.kana.address}:9153"]; + targets = ["${myvars.networking.hostsAddr.kana.ipv4}:9153"]; labels.type = "app"; labels.app = "v2ray"; } @@ -96,7 +96,7 @@ metrics_path = "/metrics"; static_configs = [ { - targets = ["${myvars.networking.hostAddress.kana.address}:10000"]; + targets = ["${myvars.networking.hostsAddr.kana.ipv4}:10000"]; labels.type = "app"; labels.app = "v2ray"; } diff --git a/hosts/k8s/k3s-prod-1-master-1/default.nix b/hosts/k8s/k3s-prod-1-master-1/default.nix index 8f7c0718..d40dd6cd 100644 --- a/hosts/k8s/k3s-prod-1-master-1/default.nix +++ b/hosts/k8s/k3s-prod-1-master-1/default.nix @@ -1,19 +1,28 @@ { + config, pkgs, myvars, mylib, ... }: let hostName = "k3s-prod-1-master-1"; # Define your hostname. - k8sLib = import ../lib.nix; - coreModule = k8sLib.gencoreModule { + + coreModule = mylib.genKubeVirtCoreModule { inherit pkgs hostName; inherit (myvars) networking; }; + k3sModule = mylib.genK3sServerModule { + inherit pkgs; + kubeconfigFile = "/home/${myvars.username}/.kube/config"; + tokenFile = config.age.secrets."k3s-prod-1-token".path; + # the first node in the cluster should be the one to initialize the cluster + clusterInit = true; + }; in { imports = (mylib.scanPaths ./.) ++ [ coreModule + k3sModule ]; } diff --git a/hosts/k8s/k3s-prod-1-master-2/default.nix b/hosts/k8s/k3s-prod-1-master-2/default.nix index 149d38d7..3d123490 100644 --- a/hosts/k8s/k3s-prod-1-master-2/default.nix +++ b/hosts/k8s/k3s-prod-1-master-2/default.nix @@ -1,19 +1,28 @@ { + config, pkgs, myvars, mylib, ... }: let hostName = "k3s-prod-1-master-2"; # define your hostname. - k8sLib = import ../lib.nix; - coreModule = k8sLib.gencoreModule { + k3sServerName = "k3s-prod-1-master-1"; + + coreModule = mylib.genKubeVirtCoreModule { inherit pkgs hostName; inherit (myvars) networking; }; + k3sModule = mylib.genK3sServerModule { + inherit pkgs; + kubeconfigFile = "/home/${myvars.username}/.kube/config"; + tokenFile = config.age.secrets."k3s-prod-1-token".path; + serverIp = myvars.networking.hostsAddr.${k3sServerName}.ipv4; + }; in { imports = (mylib.scanPaths ./.) ++ [ coreModule + k3sModule ]; } diff --git a/hosts/k8s/k3s-prod-1-master-2/k3s.nix b/hosts/k8s/k3s-prod-1-master-2/k3s.nix deleted file mode 100644 index 4b2d9c0e..00000000 --- a/hosts/k8s/k3s-prod-1-master-2/k3s.nix +++ /dev/null @@ -1,31 +0,0 @@ -{ - config, - pkgs, - myvars, - ... -}: let - serverName = "k3s-prod-1-master-1"; - serverIp = myvars.networking.hostAddress.${serverName}.address; - package = pkgs.k3s_1_29; -in { - environment.systemPackages = [package]; - services.k3s = { - inherit package; - enable = true; - role = "server"; - serverAddr = "https://${serverIp}:6443"; - tokenFile = config.age.secrets."k3s-prod-1-token".path; - # https://docs.k3s.io/cli/server - extraFlags = - " --write-kubeconfig /etc/k3s/kubeconfig.yml" - + " --write-kubeconfig-mode 644" - + " --service-node-port-range 80-32767" - + " --kube-apiserver-arg='--allow-privileged=true'" # required by kubevirt - + " --node-taint=CriticalAddonsOnly=true:NoExecute" # prevent workloads from running on the master - + " --data-dir /var/lib/rancher/k3s" - + " --disable-helm-controller" # we use fluxcd instead - + " --disable=traefik" # deploy our own ingress controller instead - + " --etcd-expose-metrics true" - + " --etcd-snapshot-schedule-cron '0 */12 * * *'"; - }; -} diff --git a/hosts/k8s/k3s-prod-1-master-3/default.nix b/hosts/k8s/k3s-prod-1-master-3/default.nix index 7269ab0c..69e9ecd9 100644 --- a/hosts/k8s/k3s-prod-1-master-3/default.nix +++ b/hosts/k8s/k3s-prod-1-master-3/default.nix @@ -1,19 +1,28 @@ { + config, pkgs, myvars, mylib, ... }: let hostName = "k3s-prod-1-master-3"; # define your hostname. - k8sLib = import ../lib.nix; - coreModule = k8sLib.gencoreModule { + k3sServerName = "k3s-prod-1-master-1"; + + coreModule = mylib.genKubeVirtCoreModule { inherit pkgs hostName; inherit (myvars) networking; }; + k3sModule = mylib.genK3sServerModule { + inherit pkgs; + kubeconfigFile = "/home/${myvars.username}/.kube/config"; + tokenFile = config.age.secrets."k3s-prod-1-token".path; + serverIp = myvars.networking.hostsAddr.${k3sServerName}.ipv4; + }; in { imports = (mylib.scanPaths ./.) ++ [ coreModule + k3sModule ]; } diff --git a/hosts/k8s/k3s-prod-1-master-3/k3s.nix b/hosts/k8s/k3s-prod-1-master-3/k3s.nix deleted file mode 100644 index 4b2d9c0e..00000000 --- a/hosts/k8s/k3s-prod-1-master-3/k3s.nix +++ /dev/null @@ -1,31 +0,0 @@ -{ - config, - pkgs, - myvars, - ... -}: let - serverName = "k3s-prod-1-master-1"; - serverIp = myvars.networking.hostAddress.${serverName}.address; - package = pkgs.k3s_1_29; -in { - environment.systemPackages = [package]; - services.k3s = { - inherit package; - enable = true; - role = "server"; - serverAddr = "https://${serverIp}:6443"; - tokenFile = config.age.secrets."k3s-prod-1-token".path; - # https://docs.k3s.io/cli/server - extraFlags = - " --write-kubeconfig /etc/k3s/kubeconfig.yml" - + " --write-kubeconfig-mode 644" - + " --service-node-port-range 80-32767" - + " --kube-apiserver-arg='--allow-privileged=true'" # required by kubevirt - + " --node-taint=CriticalAddonsOnly=true:NoExecute" # prevent workloads from running on the master - + " --data-dir /var/lib/rancher/k3s" - + " --disable-helm-controller" # we use fluxcd instead - + " --disable=traefik" # deploy our own ingress controller instead - + " --etcd-expose-metrics true" - + " --etcd-snapshot-schedule-cron '0 */12 * * *'"; - }; -} diff --git a/hosts/k8s/k3s-prod-1-worker-1/default.nix b/hosts/k8s/k3s-prod-1-worker-1/default.nix index 53a4f6d1..0a3e2e57 100644 --- a/hosts/k8s/k3s-prod-1-worker-1/default.nix +++ b/hosts/k8s/k3s-prod-1-worker-1/default.nix @@ -1,19 +1,27 @@ { + config, pkgs, myvars, mylib, ... }: let hostName = "k3s-prod-1-worker-1"; # define your hostname. - k8sLib = import ../lib.nix; - coreModule = k8sLib.gencoreModule { + k3sServerName = "k3s-prod-1-master-1"; + + coreModule = mylib.genKubeVirtCoreModule { inherit pkgs hostName; inherit (myvars) networking; }; + k3sModule = mylib.genK3sAgentModule { + inherit pkgs; + tokenFile = config.age.secrets."k3s-prod-1-token".path; + serverIp = myvars.networking.hostsAddr.${k3sServerName}.ipv4; + }; in { imports = (mylib.scanPaths ./.) ++ [ coreModule + k3sModule ]; } diff --git a/hosts/k8s/k3s-prod-1-worker-2/default.nix b/hosts/k8s/k3s-prod-1-worker-2/default.nix index 45fc4fcb..0d71f351 100644 --- a/hosts/k8s/k3s-prod-1-worker-2/default.nix +++ b/hosts/k8s/k3s-prod-1-worker-2/default.nix @@ -1,19 +1,27 @@ { + config, pkgs, myvars, mylib, ... }: let hostName = "k3s-prod-1-worker-2"; # define your hostname. - k8sLib = import ../lib.nix; - coreModule = k8sLib.gencoreModule { + k3sServerName = "k3s-prod-1-master-1"; + + coreModule = mylib.genKubeVirtCoreModule { inherit pkgs hostName; inherit (myvars) networking; }; + k3sModule = mylib.genK3sAgentModule { + inherit pkgs; + tokenFile = config.age.secrets."k3s-prod-1-token".path; + serverIp = myvars.networking.hostsAddr.${k3sServerName}.ipv4; + }; in { imports = (mylib.scanPaths ./.) ++ [ coreModule + k3sModule ]; } diff --git a/hosts/k8s/k3s-prod-1-worker-2/k3s.nix b/hosts/k8s/k3s-prod-1-worker-2/k3s.nix deleted file mode 100644 index c1b5b4e3..00000000 --- a/hosts/k8s/k3s-prod-1-worker-2/k3s.nix +++ /dev/null @@ -1,23 +0,0 @@ -{ - config, - pkgs, - myvars, - ... -}: let - serverName = "k3s-prod-1-master-1"; - serverIp = myvars.networking.hostAddress.${serverName}.address; - package = pkgs.k3s_1_29; -in { - environment.systemPackages = [package]; - services.k3s = { - inherit package; - enable = true; - role = "agent"; - serverAddr = "https://${serverIp}:6443"; - tokenFile = config.age.secrets."k3s-prod-1-token".path; - # https://docs.k3s.io/cli/agent - extraFlags = - " --node-label=node-type=worker" - + " --data-dir /var/lib/rancher/k3s"; - }; -} diff --git a/hosts/k8s/k3s-prod-1-worker-3/default.nix b/hosts/k8s/k3s-prod-1-worker-3/default.nix index 9d9299ad..16bfe7ed 100644 --- a/hosts/k8s/k3s-prod-1-worker-3/default.nix +++ b/hosts/k8s/k3s-prod-1-worker-3/default.nix @@ -1,19 +1,27 @@ { + config, pkgs, myvars, mylib, ... }: let hostName = "k3s-prod-1-worker-3"; # define your hostname. - k8sLib = import ../lib.nix; - coreModule = k8sLib.gencoreModule { + k3sServerName = "k3s-prod-1-master-1"; + + coreModule = mylib.genKubeVirtCoreModule { inherit pkgs hostName; inherit (myvars) networking; }; + k3sModule = mylib.genK3sAgentModule { + inherit pkgs; + tokenFile = config.age.secrets."k3s-prod-1-token".path; + serverIp = myvars.networking.hostsAddr.${k3sServerName}.ipv4; + }; in { imports = (mylib.scanPaths ./.) ++ [ coreModule + k3sModule ]; } diff --git a/hosts/k8s/k3s-prod-1-worker-3/k3s.nix b/hosts/k8s/k3s-prod-1-worker-3/k3s.nix deleted file mode 100644 index c1b5b4e3..00000000 --- a/hosts/k8s/k3s-prod-1-worker-3/k3s.nix +++ /dev/null @@ -1,23 +0,0 @@ -{ - config, - pkgs, - myvars, - ... -}: let - serverName = "k3s-prod-1-master-1"; - serverIp = myvars.networking.hostAddress.${serverName}.address; - package = pkgs.k3s_1_29; -in { - environment.systemPackages = [package]; - services.k3s = { - inherit package; - enable = true; - role = "agent"; - serverAddr = "https://${serverIp}:6443"; - tokenFile = config.age.secrets."k3s-prod-1-token".path; - # https://docs.k3s.io/cli/agent - extraFlags = - " --node-label=node-type=worker" - + " --data-dir /var/lib/rancher/k3s"; - }; -} diff --git a/hosts/k8s/kubevirt-shoryu/default.nix b/hosts/k8s/kubevirt-shoryu/default.nix index bf246180..d108f8e2 100644 --- a/hosts/k8s/kubevirt-shoryu/default.nix +++ b/hosts/k8s/kubevirt-shoryu/default.nix @@ -1,4 +1,5 @@ { + config, pkgs, mylib, myvars, @@ -7,17 +8,25 @@ }: let # MoreFine - S500Plus hostName = "kubevirt-shoryu"; # Define your hostname. - k8sLib = import ../lib.nix; - coreModule = k8sLib.gencoreModule { + + coreModule = mylib.genKubeVirtCoreModule { inherit pkgs hostName; inherit (myvars) networking; }; + k3sModule = mylib.genK3sServerModule { + inherit pkgs; + kubeconfigFile = "/home/${myvars.username}/.kube/config"; + tokenFile = config.age.secrets."k3s-prod-1-token".path; + # the first node in the cluster should be the one to initialize the cluster + clusterInit = true; + }; in { imports = (mylib.scanPaths ./.) ++ [ - coreModule disko.nixosModules.default ../disko-config/kubevirt-disko-fs.nix + coreModule + k3sModule ]; } diff --git a/hosts/k8s/kubevirt-shoryu/k3s.nix b/hosts/k8s/kubevirt-shoryu/k3s.nix deleted file mode 100644 index 81ec1e95..00000000 --- a/hosts/k8s/kubevirt-shoryu/k3s.nix +++ /dev/null @@ -1,42 +0,0 @@ -{ - config, - pkgs, - myvars, - ... -}: let - package = pkgs.k3s_1_29; -in { - environment.systemPackages = with pkgs; [ - package - k9s - kubectl - istioctl - kubernetes-helm - - skopeo - dive # explore docker layers - ]; - services.k3s = { - inherit package; - enable = true; - - # Initialize HA cluster using an embedded etcd datastore. - # If you are configuring an HA cluster with an embedded etcd, - # the 1st server must have `clusterInit = true` - # and other servers must connect to it using serverAddr. - clusterInit = true; - role = "server"; - tokenFile = "/run/media/nixos_k3s/kubevirt-k3s-token"; - # https://docs.k3s.io/cli/server - extraFlags = - " --write-kubeconfig /etc/k3s/kubeconfig.yml" - + " --write-kubeconfig-mode 644" - + " --service-node-port-range 80-32767" - + " --kube-apiserver-arg='--allow-privileged=true'" # required by kubevirt - + " --data-dir /var/lib/rancher/k3s" - + " --disable-helm-controller" # we use fluxcd instead - + " --disable=traefik" # deploy our own ingress controller instead - + " --etcd-expose-metrics true" - + " --etcd-snapshot-schedule-cron '0 */12 * * *'"; - }; -} diff --git a/hosts/k8s/kubevirt-shushou/default.nix b/hosts/k8s/kubevirt-shushou/default.nix index b02a23e1..2de3c6a1 100644 --- a/hosts/k8s/kubevirt-shushou/default.nix +++ b/hosts/k8s/kubevirt-shushou/default.nix @@ -1,4 +1,5 @@ { + config, pkgs, mylib, myvars, @@ -6,17 +7,25 @@ ... }: let hostName = "kubevirt-shushou"; # Define your hostname. - k8sLib = import ../lib.nix; - coreModule = k8sLib.gencoreModule { + k3sServerName = "kubevirt-shoryu"; + + coreModule = mylib.genKubeVirtCoreModule { inherit pkgs hostName; inherit (myvars) networking; }; + k3sModule = mylib.genK3sServerModule { + inherit pkgs; + kubeconfigFile = "/home/${myvars.username}/.kube/config"; + tokenFile = config.age.secrets."k3s-prod-1-token".path; + serverIp = myvars.networking.hostsAddr.${k3sServerName}.ipv4; + }; in { imports = (mylib.scanPaths ./.) ++ [ - coreModule disko.nixosModules.default ../disko-config/kubevirt-disko-fs.nix + coreModule + k3sModule ]; } diff --git a/hosts/k8s/kubevirt-shushou/k3s.nix b/hosts/k8s/kubevirt-shushou/k3s.nix deleted file mode 100644 index 81ec1e95..00000000 --- a/hosts/k8s/kubevirt-shushou/k3s.nix +++ /dev/null @@ -1,42 +0,0 @@ -{ - config, - pkgs, - myvars, - ... -}: let - package = pkgs.k3s_1_29; -in { - environment.systemPackages = with pkgs; [ - package - k9s - kubectl - istioctl - kubernetes-helm - - skopeo - dive # explore docker layers - ]; - services.k3s = { - inherit package; - enable = true; - - # Initialize HA cluster using an embedded etcd datastore. - # If you are configuring an HA cluster with an embedded etcd, - # the 1st server must have `clusterInit = true` - # and other servers must connect to it using serverAddr. - clusterInit = true; - role = "server"; - tokenFile = "/run/media/nixos_k3s/kubevirt-k3s-token"; - # https://docs.k3s.io/cli/server - extraFlags = - " --write-kubeconfig /etc/k3s/kubeconfig.yml" - + " --write-kubeconfig-mode 644" - + " --service-node-port-range 80-32767" - + " --kube-apiserver-arg='--allow-privileged=true'" # required by kubevirt - + " --data-dir /var/lib/rancher/k3s" - + " --disable-helm-controller" # we use fluxcd instead - + " --disable=traefik" # deploy our own ingress controller instead - + " --etcd-expose-metrics true" - + " --etcd-snapshot-schedule-cron '0 */12 * * *'"; - }; -} diff --git a/hosts/k8s/kubevirt-youko/default.nix b/hosts/k8s/kubevirt-youko/default.nix index 0a641285..137c391a 100644 --- a/hosts/k8s/kubevirt-youko/default.nix +++ b/hosts/k8s/kubevirt-youko/default.nix @@ -1,4 +1,5 @@ { + config, pkgs, mylib, myvars, @@ -6,17 +7,25 @@ ... }: let hostName = "kubevirt-youko"; # Define your hostname. - k8sLib = import ../lib.nix; - coreModule = k8sLib.gencoreModule { + k3sServerName = "kubevirt-shoryu"; + + coreModule = mylib.genKubeVirtCoreModule { inherit pkgs hostName; inherit (myvars) networking; }; + k3sModule = mylib.genK3sServerModule { + inherit pkgs; + kubeconfigFile = "/home/${myvars.username}/.kube/config"; + tokenFile = config.age.secrets."k3s-prod-1-token".path; + serverIp = myvars.networking.hostsAddr.${k3sServerName}.ipv4; + }; in { imports = (mylib.scanPaths ./.) ++ [ - coreModule disko.nixosModules.default ../disko-config/kubevirt-disko-fs.nix + coreModule + k3sModule ]; } diff --git a/hosts/k8s/kubevirt-youko/k3s.nix b/hosts/k8s/kubevirt-youko/k3s.nix deleted file mode 100644 index 81ec1e95..00000000 --- a/hosts/k8s/kubevirt-youko/k3s.nix +++ /dev/null @@ -1,42 +0,0 @@ -{ - config, - pkgs, - myvars, - ... -}: let - package = pkgs.k3s_1_29; -in { - environment.systemPackages = with pkgs; [ - package - k9s - kubectl - istioctl - kubernetes-helm - - skopeo - dive # explore docker layers - ]; - services.k3s = { - inherit package; - enable = true; - - # Initialize HA cluster using an embedded etcd datastore. - # If you are configuring an HA cluster with an embedded etcd, - # the 1st server must have `clusterInit = true` - # and other servers must connect to it using serverAddr. - clusterInit = true; - role = "server"; - tokenFile = "/run/media/nixos_k3s/kubevirt-k3s-token"; - # https://docs.k3s.io/cli/server - extraFlags = - " --write-kubeconfig /etc/k3s/kubeconfig.yml" - + " --write-kubeconfig-mode 644" - + " --service-node-port-range 80-32767" - + " --kube-apiserver-arg='--allow-privileged=true'" # required by kubevirt - + " --data-dir /var/lib/rancher/k3s" - + " --disable-helm-controller" # we use fluxcd instead - + " --disable=traefik" # deploy our own ingress controller instead - + " --etcd-expose-metrics true" - + " --etcd-snapshot-schedule-cron '0 */12 * * *'"; - }; -} diff --git a/hosts/k8s/lib.nix b/hosts/k8s/lib.nix deleted file mode 100644 index ab9999be..00000000 --- a/hosts/k8s/lib.nix +++ /dev/null @@ -1,86 +0,0 @@ -{ - gencoreModule = { - pkgs, - hostName, - networking, - ... - }: let - hostAddress = networking.hostAddress.${hostName}; - in { - # supported file systems, so we can mount any removable disks with these filesystems - boot.supportedFilesystems = [ - "ext4" - "btrfs" - "xfs" - #"zfs" - "ntfs" - "fat" - "vfat" - "exfat" - "nfs" # required by longhorn - "cifs" # mount windows share - ]; - - boot.kernelModules = ["kvm-amd" "vfio-pci"]; - boot.extraModprobeConfig = "options kvm_amd nested=1"; # for amd cpu - - environment.systemPackages = with pkgs; [ - # Validate Hardware Virtualization Support via: - # virt-host-validate qemu - libvirt - - # used by kubernetes' ovs-cni plugin - # https://github.com/k8snetworkplumbingwg/multus-cni - multus-cni - ]; - - # Enable the Open vSwitch as a systemd service - # It's required by kubernetes' ovs-cni plugin. - virtualisation.vswitch = { - enable = true; - # reset the Open vSwitch configuration database to a default configuration on every start of the systemd ovsdb.service - resetOnStart = false; - }; - networking.vswitches = { - # https://github.com/k8snetworkplumbingwg/ovs-cni/blob/main/docs/demo.md - ovsbr1 = { - interfaces = { - # Attach the interfaces to OVS bridge - # This interface should not used by the host itself! - ens18 = {}; - }; - }; - }; - - # Workaround for longhorn running on NixOS - # https://github.com/longhorn/longhorn/issues/2166 - systemd.tmpfiles.rules = [ - "L+ /usr/local/bin - - - - /run/current-system/sw/bin/" - ]; - # Longhorn uses open-iscsi to create block devices. - services.openiscsi = { - name = "iqn.2020-08.org.linux-iscsi.initiatorhost:${hostName}"; - enable = true; - }; - - networking = { - inherit hostName; - inherit (networking) defaultGateway nameservers; - - networkmanager.enable = false; - # Set the host's address on the OVS bridge interface instead of the physical interface! - interfaces.ovsbr1 = { - useDHCP = false; - ipv4.addresses = [hostAddress]; - }; - }; - - # This value determines the NixOS release from which the default - # settings for stateful data, like file locations and database versions - # on your system were taken. It‘s perfectly fine and recommended to leave - # this value at the release version of the first install of this system. - # Before changing this value read the documentation for this option - # (e.g. man configuration.nix or on https://nixos.org/nixos/options.html). - system.stateVersion = "23.11"; # Did you read the comment? - }; -} diff --git a/hosts/rolling-girls-nozomi/default.nix b/hosts/rolling-girls-nozomi/default.nix index 76211ea2..b524d853 100644 --- a/hosts/rolling-girls-nozomi/default.nix +++ b/hosts/rolling-girls-nozomi/default.nix @@ -10,7 +10,6 @@ ############################################################# let hostName = "nozomi"; # Define your hostname. - hostAddress = myvars.networking.hostAddress.${hostName}; in { imports = [ # import the licheepi4a module, which contains the configuration for bootloader/kernel/firmware @@ -23,6 +22,7 @@ in { networking = { inherit hostName; inherit (myvars.networking) defaultGateway nameservers; + inherit (myvars.networking.hostsInterface.${hostName}) interfaces; wireless = { # https://wiki.archlinux.org/title/wpa_supplicant @@ -46,11 +46,6 @@ in { # proxy.default = "http://user:password@proxy:port/"; # proxy.noProxy = "127.0.0.1,localhost,internal.domain"; - # LPI4A's wireless interface - interfaces.wlan0 = { - useDHCP = false; - ipv4.addresses = [hostAddress]; - }; # LPI4A's first ethernet interface # interfaces.end0 = { # useDHCP = false; diff --git a/hosts/rolling-girls-yukina/default.nix b/hosts/rolling-girls-yukina/default.nix index 2e5a4911..0508ac61 100644 --- a/hosts/rolling-girls-yukina/default.nix +++ b/hosts/rolling-girls-yukina/default.nix @@ -10,7 +10,6 @@ ############################################################# let hostName = "yukina"; # Define your hostname. - hostAddress = myvars.networking.hostAddress.${hostName}; in { imports = [ # import the licheepi4a module, which contains the configuration for bootloader/kernel/firmware @@ -23,6 +22,7 @@ in { networking = { inherit hostName; inherit (myvars.networking) defaultGateway nameservers; + inherit (myvars.networking.hostsInterface.${hostName}) interfaces; wireless = { # https://wiki.archlinux.org/title/wpa_supplicant @@ -46,11 +46,6 @@ in { # proxy.default = "http://user:password@proxy:port/"; # proxy.noProxy = "127.0.0.1,localhost,internal.domain"; - # LPI4A's wireless interface - interfaces.wlan0 = { - useDHCP = false; - ipv4.addresses = [hostAddress]; - }; # LPI4A's first ethernet interface # interfaces.end0 = { # useDHCP = false; diff --git a/lib/default.nix b/lib/default.nix index 6cbd7c06..58f36d61 100644 --- a/lib/default.nix +++ b/lib/default.nix @@ -2,7 +2,13 @@ colmenaSystem = import ./colmenaSystem.nix; macosSystem = import ./macosSystem.nix; nixosSystem = import ./nixosSystem.nix; + attrs = import ./attrs.nix {inherit lib;}; + + genKubeVirtCoreModule = import ./genKubeVirtCoreModule.nix; + genK3sServerModule = import ./genK3sServerModule.nix; + genK3sAgentModule = import ./genK3sAgentModule.nix; + # use path relative to the root of the project relativeToRoot = lib.path.append ../.; scanPaths = path: diff --git a/hosts/k8s/k3s-prod-1-worker-1/k3s.nix b/lib/genK3sAgentModule.nix similarity index 62% rename from hosts/k8s/k3s-prod-1-worker-1/k3s.nix rename to lib/genK3sAgentModule.nix index c1b5b4e3..fabcf12d 100644 --- a/hosts/k8s/k3s-prod-1-worker-1/k3s.nix +++ b/lib/genK3sAgentModule.nix @@ -1,20 +1,18 @@ { - config, pkgs, - myvars, + serverIp, + tokenFile, ... }: let - serverName = "k3s-prod-1-master-1"; - serverIp = myvars.networking.hostAddress.${serverName}.address; package = pkgs.k3s_1_29; in { environment.systemPackages = [package]; services.k3s = { - inherit package; enable = true; + inherit package tokenFile; + role = "agent"; serverAddr = "https://${serverIp}:6443"; - tokenFile = config.age.secrets."k3s-prod-1-token".path; # https://docs.k3s.io/cli/agent extraFlags = " --node-label=node-type=worker" diff --git a/hosts/k8s/k3s-prod-1-master-1/k3s.nix b/lib/genK3sServerModule.nix similarity index 52% rename from hosts/k8s/k3s-prod-1-master-1/k3s.nix rename to lib/genK3sServerModule.nix index 5d93a76f..e88bd85a 100644 --- a/hosts/k8s/k3s-prod-1-master-1/k3s.nix +++ b/lib/genK3sServerModule.nix @@ -1,7 +1,13 @@ { - config, pkgs, - myvars, + kubeconfigFile, + tokenFile, + # Initialize HA cluster using an embedded etcd datastore. + # If you are configuring an HA cluster with an embedded etcd, + # the 1st server must have `clusterInit = true` + # and other servers must connect to it using `serverAddr`. + serverIp ? null, + clusterInit ? (serverIp == null), ... }: let package = pkgs.k3s_1_29; @@ -16,28 +22,31 @@ in { skopeo dive # explore docker layers ]; - services.k3s = { - inherit package; - enable = true; - # Initialize HA cluster using an embedded etcd datastore. - # If you are configuring an HA cluster with an embedded etcd, - # the 1st server must have `clusterInit = true` - # and other servers must connect to it using serverAddr. - clusterInit = true; + services.k3s = { + enable = true; + inherit package tokenFile clusterInit; + serverAddr = + if clusterInit + then "" + else "https://${serverIp}:6443"; + role = "server"; - tokenFile = config.age.secrets."k3s-prod-1-token".path; # https://docs.k3s.io/cli/server extraFlags = - " --write-kubeconfig /home/${myvars.username}/.kube/config" + " --write-kubeconfig ${kubeconfigFile}" + " --write-kubeconfig-mode 644" + " --service-node-port-range 80-32767" + " --kube-apiserver-arg='--allow-privileged=true'" # required by kubevirt + " --node-taint=CriticalAddonsOnly=true:NoExecute" # prevent workloads from running on the master + " --data-dir /var/lib/rancher/k3s" + + " --etcd-expose-metrics true" + + " --etcd-snapshot-schedule-cron '0 */12 * * *'" + # disable some features we don't need + " --disable-helm-controller" # we use fluxcd instead + " --disable=traefik" # deploy our own ingress controller instead - + " --etcd-expose-metrics true" - + " --etcd-snapshot-schedule-cron '0 */12 * * *'"; + + " --disable=servicelb" # we use kube-vip instead + + " --flannel-backend=none" # we use cilium instead + + " --disable-network-policy"; }; } diff --git a/lib/genKubeVirtCoreModule.nix b/lib/genKubeVirtCoreModule.nix new file mode 100644 index 00000000..feed2c0f --- /dev/null +++ b/lib/genKubeVirtCoreModule.nix @@ -0,0 +1,78 @@ +{ + pkgs, + hostName, + networking, + ... +}: let + inherit (networking.hostsAddr.${hostName}) iface; +in { + # supported file systems, so we can mount any removable disks with these filesystems + boot.supportedFilesystems = [ + "ext4" + "btrfs" + "xfs" + #"zfs" + "ntfs" + "fat" + "vfat" + "exfat" + "nfs" # required by longhorn + "cifs" # mount windows share + ]; + + boot.kernelModules = ["kvm-amd" "vfio-pci"]; + boot.extraModprobeConfig = "options kvm_amd nested=1"; # for amd cpu + + environment.systemPackages = with pkgs; [ + # Validate Hardware Virtualization Support via: + # virt-host-validate qemu + libvirt + + # used by kubernetes' ovs-cni plugin + # https://github.com/k8snetworkplumbingwg/multus-cni + multus-cni + ]; + + # Workaround for longhorn running on NixOS + # https://github.com/longhorn/longhorn/issues/2166 + systemd.tmpfiles.rules = [ + "L+ /usr/local/bin - - - - /run/current-system/sw/bin/" + ]; + # Longhorn uses open-iscsi to create block devices. + services.openiscsi = { + name = "iqn.2020-08.org.linux-iscsi.initiatorhost:${hostName}"; + enable = true; + }; + + # Enable the Open vSwitch as a systemd service + # It's required by kubernetes' ovs-cni plugin. + virtualisation.vswitch = { + enable = true; + # reset the Open vSwitch configuration database to a default configuration on every start of the systemd ovsdb.service + resetOnStart = false; + }; + networking.vswitches = { + # https://github.com/k8snetworkplumbingwg/ovs-cni/blob/main/docs/demo.md + ovsbr1 = { + # Attach the interfaces to OVS bridge + # This interface should not used by the host itself! + interfaces.${iface} = {}; + }; + }; + networking = { + inherit hostName; + inherit (networking) defaultGateway nameservers; + + networkmanager.enable = false; + # Set the host's address on the OVS bridge interface instead of the physical interface! + interfaces.ovsbr1 = networking.hostsInterface.${hostName}.interfaces.${iface}; + }; + + # This value determines the NixOS release from which the default + # settings for stateful data, like file locations and database versions + # on your system were taken. It‘s perfectly fine and recommended to leave + # this value at the release version of the first install of this system. + # Before changing this value read the documentation for this option + # (e.g. man configuration.nix or on https://nixos.org/nixos/options.html). + system.stateVersion = "23.11"; # Did you read the comment? +} diff --git a/outputs/README.md b/outputs/README.md index d0012fba..09c94622 100644 --- a/outputs/README.md +++ b/outputs/README.md @@ -104,7 +104,6 @@ All the outputs of this flake are defined here. ├── nixos-tests ├── src # every host has its own file in this directory │   ├── 12kingdoms-shoukei.nix - │   ├── homelab-tailscale-gw.nix │   ├── idols-ai.nix │   ├── idols-aquamarine.nix │   ├── idols-kana.nix diff --git a/outputs/x86_64-linux/src/homelab-tailscale-gw.nix b/outputs/x86_64-linux/src/homelab-tailscale-gw.nix deleted file mode 100644 index 0c4723dd..00000000 --- a/outputs/x86_64-linux/src/homelab-tailscale-gw.nix +++ /dev/null @@ -1,37 +0,0 @@ -{ - # NOTE: the args not used in this file CAN NOT be removed! - # because haumea pass argument lazily, - # and these arguments are used in the functions like `mylib.nixosSystem`, `mylib.colmenaSystem`, etc. - inputs, - lib, - mylib, - myvars, - system, - genSpecialArgs, - ... -} @ args: let - name = "tailscale-gw"; - tags = [name "homelab-network"]; - ssh-user = "root"; - - modules = { - nixos-modules = map mylib.relativeToRoot [ - # common - "secrets/nixos.nix" - "modules/nixos/server/server.nix" - "modules/nixos/server/proxmox-hardware-configuration.nix" - # host specific - "hosts/homelab-${name}" - ]; - }; - - systemArgs = modules // args; -in { - nixosConfigurations.${name} = mylib.nixosSystem systemArgs; - - colmena.${name} = - mylib.colmenaSystem (systemArgs // {inherit tags ssh-user;}); - - # generate proxmox image for virtual machines without desktop environment - packages.${name} = inputs.self.nixosConfigurations.${name}.config.formats.proxmox; -} diff --git a/vars/networking.nix b/vars/networking.nix index 339ca3f6..23f08bfb 100644 --- a/vars/networking.nix +++ b/vars/networking.nix @@ -7,30 +7,116 @@ ]; prefixLength = 24; - hostAddress = - lib.attrsets.mapAttrs - (name: address: {inherit prefixLength address;}) - { - "ai" = "192.168.5.100"; - "aquamarine" = "192.168.5.101"; - "ruby" = "192.168.5.102"; - "kana" = "192.168.5.103"; - "nozomi" = "192.168.5.104"; - "yukina" = "192.168.5.105"; - "chiaya" = "192.168.5.106"; - "suzu" = "192.168.5.107"; - "k3s-prod-1-master-1" = "192.168.5.108"; - "k3s-prod-1-master-2" = "192.168.5.109"; - "k3s-prod-1-master-3" = "192.168.5.110"; - "k3s-prod-1-worker-1" = "192.168.5.111"; - "k3s-prod-1-worker-2" = "192.168.5.112"; - "k3s-prod-1-worker-3" = "192.168.5.113"; - "kubevirt-shoryu" = "192.168.5.176"; - "kubevirt-shushou" = "192.168.5.177"; - "kubevirt-youko" = "192.168.5.178"; - "rakushun" = "192.168.5.179"; - "tailscale-gw" = "192.168.5.192"; + hostsAddr = { + # Homelab's Physical Machines (KubeVirt Nodes) + kubevirt-shoryu = { + iface = "eno1"; + ipv4 = "192.168.5.181"; }; + kubevirt-shushou = { + iface = "eno1"; + ipv4 = "192.168.5.182"; + }; + kubevirt-youko = { + iface = "eno1"; + ipv4 = "192.168.5.183"; + }; + + # Other VMs and Physical Machines + ai = { + # Desktop PC + iface = "enp5s0"; + ipv4 = "192.168.5.100"; + }; + aquamarine = { + # VM + iface = "ens18"; + ipv4 = "192.168.5.101"; + }; + ruby = { + # VM + iface = "ens18"; + ipv4 = "192.168.5.102"; + }; + kana = { + # VM + iface = "ens18"; + ipv4 = "192.168.5.103"; + }; + nozomi = { + # LicheePi 4A's wireless iterface - RISC-V + iface = "wlan0"; + ipv4 = "192.168.5.104"; + }; + yukina = { + # LicheePi 4A's wireless iterface - RISC-V + iface = "wlan0"; + ipv4 = "192.168.5.105"; + }; + chiaya = { + # VM + iface = "ens18"; + ipv4 = "192.168.5.106"; + }; + suzu = { + # Orange Pi 5 - ARM + iface = "end1"; + ipv4 = "192.168.5.107"; + }; + rakushun = { + # Orange Pi 5 - ARM + # RJ45 port 1 - enP4p65s0 + # RJ45 port 2 - enP3p49s0 + iface = "enP4p65s0"; + ipv4 = "192.168.5.179"; + }; + + k3s-prod-1-master-1 = { + # VM + iface = "ens18"; + ipv4 = "192.168.5.108"; + }; + k3s-prod-1-master-2 = { + # VM + iface = "ens18"; + ipv4 = "192.168.5.109"; + }; + k3s-prod-1-master-3 = { + # VM + iface = "ens18"; + ipv4 = "192.168.5.110"; + }; + k3s-prod-1-worker-1 = { + # VM + iface = "ens18"; + ipv4 = "192.168.5.111"; + }; + k3s-prod-1-worker-2 = { + # VM + iface = "ens18"; + ipv4 = "192.168.5.112"; + }; + k3s-prod-1-worker-3 = { + # VM + iface = "ens18"; + ipv4 = "192.168.5.113"; + }; + }; + + hostsInterface = + lib.attrsets.mapAttrs + ( + key: val: { + interfaces."${val.iface}" = { + useDHCP = false; + ipv4.addresses = [{ + inherit prefixLength; + address = val.ipv4; + }]; + }; + } + ) + hostsAddr; ssh = { # define the host alias for remote builders @@ -47,15 +133,15 @@ # ''; extraConfig = lib.attrsets.foldlAttrs - (acc: host: value: + (acc: host: val: acc + '' Host ${host} - HostName ${value.address} + HostName ${val.ipv4} Port 22 '') "" - hostAddress; + hostsAddr; # define the host key for remote builders so that nix can verify all the remote builders # this config will be written to /etc/ssh/ssh_known_hosts @@ -68,7 +154,7 @@ # => { x = "bar-a"; y = "bar-b"; } lib.attrsets.mapAttrs (host: value: { - hostNames = [host hostAddress.${host}.address]; + hostNames = [host hostsAddr.${host}.ipv4]; publicKey = value.publicKey; }) {