From 2319300f48b9a971e4c06d9481fcaa7a0ae4d8f8 Mon Sep 17 00:00:00 2001 From: Ryan Yin Date: Sat, 23 Mar 2024 02:05:54 +0800 Subject: [PATCH] feat: add a test k3s cluster running on kubevirt --- Justfile | 49 ++++++++++--------- hosts/12kingdoms-rakushun/caddy.nix | 17 +++++++ hosts/README.md | 3 ++ hosts/k8s/k3s-test-1-master-1/default.nix | 28 +++++++++++ hosts/k8s/k3s-test-1-master-2/default.nix | 28 +++++++++++ hosts/k8s/k3s-test-1-master-3/default.nix | 28 +++++++++++ .../kubevirt-hardware-configuration.nix | 34 +++++++++++++ .../x86_64-linux/src/k3s-test-1-master-1.nix | 44 +++++++++++++++++ .../x86_64-linux/src/k3s-test-1-master-2.nix | 41 ++++++++++++++++ .../x86_64-linux/src/k3s-test-1-master-3.nix | 41 ++++++++++++++++ secrets/nixos.nix | 6 +++ utils.nu | 19 +++++++ vars/networking.nix | 26 ++++++++-- 13 files changed, 338 insertions(+), 26 deletions(-) create mode 100644 hosts/k8s/k3s-test-1-master-1/default.nix create mode 100644 hosts/k8s/k3s-test-1-master-2/default.nix create mode 100644 hosts/k8s/k3s-test-1-master-3/default.nix create mode 100644 modules/nixos/server/kubevirt-hardware-configuration.nix create mode 100644 outputs/x86_64-linux/src/k3s-test-1-master-1.nix create mode 100644 outputs/x86_64-linux/src/k3s-test-1-master-2.nix create mode 100644 outputs/x86_64-linux/src/k3s-test-1-master-3.nix diff --git a/Justfile b/Justfile index 6a4deaa7..f26acd58 100644 --- a/Justfile +++ b/Justfile @@ -27,10 +27,11 @@ s-hypr mode="default": use utils.nu *; \ nixos-switch shoukei-hyprland {{mode}} - +# Run eval tests test: nix eval .#evalTests --show-trace --print-build-logs --verbose +# update all the flake inputs up: nix flake update @@ -39,21 +40,25 @@ up: upp input: nix flake lock --update-input {{input}} +# List all generations of the system profile history: nix profile history --profile /nix/var/nix/profiles/system +# Open a nix shell with the flake repl: nix repl -f flake:nixpkgs +# remove all generations older than 7 days clean: - # remove all generations older than 7 days sudo nix profile wipe-history --profile /nix/var/nix/profiles/system --older-than 7d +# Garbage collect all unused nix store entries gc: # garbage collect all unused nix store entries sudo nix store gc --debug sudo nix-collect-garbage --delete-old +# Remove all reflog entries and prune unreachable objects gitgc: git reflog expire --expire-unreachable=now --all git gc --prune=now @@ -72,46 +77,44 @@ darwin-rollback: use utils.nu *; \ darwin-rollback +# Deploy to harmonica(macOS host) ha mode="default": use utils.nu *; \ darwin-build "harmonica" {{mode}}; \ darwin-switch "harmonica" {{mode}} +# Depoly to fern(macOS host) fe mode="default": darwin-set-proxy use utils.nu *; \ darwin-build "fern" {{mode}}; \ darwin-switch "fern" {{mode}} +# Reload yabai and skhd(macOS) yabai-reload: launchctl kickstart -k "gui/502/org.nixos.yabai"; launchctl kickstart -k "gui/502/org.nixos.skhd"; -############################################################################ -# -# Homelab - NixOS servers running on bare metal -# -############################################################################ - -virt: - colmena apply --on '@virt-*' --verbose --show-trace - -shoryu: - colmena apply --on '@shoryu' --verbose --show-trace - -shushou: - colmena apply --on '@shushou' --verbose --show-trace - -youko: - colmena apply --on '@youko' --verbose --show-trace - - ############################################################################ # # Homelab - Virtual Machines running on Kubevirt # ############################################################################ +# Remote deployment via colmena +col tag: + colmena apply --on '@{{tag}}' --verbose --show-trace + +# Build and upload a vm image +upload-vm name mode="default": + use utils.nu *; \ + upload-vm {{name}} {{mode}} + +# Deploy all the KubeVirt nodes(Physical machines running KubeVirt) lab: + colmena apply --on '@virt-*' --verbose --show-trace + +# Deploy all the VMs running on KubeVirt +vm: colmena apply --on '@homelab-*' --verbose --show-trace aqua: @@ -144,6 +147,8 @@ master: worker: colmena apply --on '@k3s-prod-1-worker-*' --verbose --show-trace +k3s-test: + colmena apply --on '@k3s-test-*' --verbose --show-trace ############################################################################ # @@ -167,7 +172,7 @@ yukina: ############################################################################ aarch: - colmena apply --on '@aarch' --verbose --show-trace + colmena apply --on '@aarch' --build-on-target --verbose --show-trace suzu: colmena apply --on '@suzu' --build-on-target --verbose --show-trace diff --git a/hosts/12kingdoms-rakushun/caddy.nix b/hosts/12kingdoms-rakushun/caddy.nix index fbd200bb..d89a942c 100644 --- a/hosts/12kingdoms-rakushun/caddy.nix +++ b/hosts/12kingdoms-rakushun/caddy.nix @@ -23,6 +23,23 @@ encode zstd gzip reverse_proxy http://localhost:3000 ''; + + # https://caddyserver.com/docs/caddyfile/directives/file_server + virtualHosts."http://file.writefor.fun".extraConfig = '' + root * /var/lib/caddy/fileserver/ + encode zstd gzip + file_server browse { + hide .git + precompressed zstd br gzip + } + ''; }; networking.firewall.allowedTCPPorts = [80 443]; + + # Create Directories + systemd.tmpfiles.rules = [ + "d /var/lib/caddy/fileserver/ 0755 caddy caddy" + # directory for virual machine's images + "d /var/lib/caddy/fileserver/vms 0755 caddy caddy" + ]; } diff --git a/hosts/README.md b/hosts/README.md index 3ffa9e47..6bf895f8 100644 --- a/hosts/README.md +++ b/hosts/README.md @@ -38,6 +38,9 @@ 4. [Optional] Add a new integration test file under `outputs//integration-tests/.nix` to test whether the new host's nix config can be built and deployed correctly. +1. Under `vars/networking.nix` + 1. Add the new host's static IP address. + 1. Skip this step if the new host is not in the local network or is a mobile device. ## idols - Oshi no Ko diff --git a/hosts/k8s/k3s-test-1-master-1/default.nix b/hosts/k8s/k3s-test-1-master-1/default.nix new file mode 100644 index 00000000..77eaebe6 --- /dev/null +++ b/hosts/k8s/k3s-test-1-master-1/default.nix @@ -0,0 +1,28 @@ +{ + config, + pkgs, + myvars, + mylib, + ... +}: let + hostName = "k3s-test-1-master-1"; # Define your hostname. + + coreModule = mylib.genKubeVirtCoreModule { + inherit pkgs hostName; + inherit (myvars) networking; + }; + k3sModule = mylib.genK3sServerModule { + inherit pkgs; + kubeconfigFile = "/home/${myvars.username}/.kube/config"; + tokenFile = config.age.secrets."k3s-prod-1-token".path; + # the first node in the cluster should be the one to initialize the cluster + clusterInit = true; + }; +in { + imports = + (mylib.scanPaths ./.) + ++ [ + coreModule + k3sModule + ]; +} diff --git a/hosts/k8s/k3s-test-1-master-2/default.nix b/hosts/k8s/k3s-test-1-master-2/default.nix new file mode 100644 index 00000000..d57a3dc5 --- /dev/null +++ b/hosts/k8s/k3s-test-1-master-2/default.nix @@ -0,0 +1,28 @@ +{ + config, + pkgs, + myvars, + mylib, + ... +}: let + hostName = "k3s-test-1-master-2"; # define your hostname. + k3sServerName = "k3s-test-1-master-1"; + + coreModule = mylib.genKubeVirtCoreModule { + inherit pkgs hostName; + inherit (myvars) networking; + }; + k3sModule = mylib.genK3sServerModule { + inherit pkgs; + kubeconfigFile = "/home/${myvars.username}/.kube/config"; + tokenFile = config.age.secrets."k3s-prod-1-token".path; + serverIp = myvars.networking.hostsAddr.${k3sServerName}.ipv4; + }; +in { + imports = + (mylib.scanPaths ./.) + ++ [ + coreModule + k3sModule + ]; +} diff --git a/hosts/k8s/k3s-test-1-master-3/default.nix b/hosts/k8s/k3s-test-1-master-3/default.nix new file mode 100644 index 00000000..7680b4b7 --- /dev/null +++ b/hosts/k8s/k3s-test-1-master-3/default.nix @@ -0,0 +1,28 @@ +{ + config, + pkgs, + myvars, + mylib, + ... +}: let + hostName = "k3s-test-1-master-3"; # define your hostname. + k3sServerName = "k3s-test-1-master-1"; + + coreModule = mylib.genKubeVirtCoreModule { + inherit pkgs hostName; + inherit (myvars) networking; + }; + k3sModule = mylib.genK3sServerModule { + inherit pkgs; + kubeconfigFile = "/home/${myvars.username}/.kube/config"; + tokenFile = config.age.secrets."k3s-prod-1-token".path; + serverIp = myvars.networking.hostsAddr.${k3sServerName}.ipv4; + }; +in { + imports = + (mylib.scanPaths ./.) + ++ [ + coreModule + k3sModule + ]; +} diff --git a/modules/nixos/server/kubevirt-hardware-configuration.nix b/modules/nixos/server/kubevirt-hardware-configuration.nix new file mode 100644 index 00000000..76565629 --- /dev/null +++ b/modules/nixos/server/kubevirt-hardware-configuration.nix @@ -0,0 +1,34 @@ +{modulesPath, ...}: +############################################################################## +# +# Template for KubeVirt's VM, mainly based on: +# https://github.com/NixOS/nixpkgs/blob/master/nixos/modules/virtualisation/kubevirt.nix +# +# We write our hardware-configuration.nix, so that we can do some customization more easily. +# +# the url above is used by `nixos-generator` to generate the KubeVirt's qcow2 image file. +# +############################################################################## +{ + imports = [ + "${toString modulesPath}/profiles/qemu-guest.nix" + ]; + + config = { + fileSystems."/" = { + device = "/dev/disk/by-label/nixos"; + fsType = "ext4"; + autoResize = true; + }; + + boot.growPartition = true; + boot.kernelParams = [ "console=ttyS0" ]; + boot.loader.grub.device = "/dev/vda"; + boot.loader.timeout = 0; + + services.qemuGuest.enable = true; + services.openssh.enable = true; + services.cloud-init.enable = true; + systemd.services."serial-getty@ttyS0".enable = true; + }; +} diff --git a/outputs/x86_64-linux/src/k3s-test-1-master-1.nix b/outputs/x86_64-linux/src/k3s-test-1-master-1.nix new file mode 100644 index 00000000..5c8e0fe8 --- /dev/null +++ b/outputs/x86_64-linux/src/k3s-test-1-master-1.nix @@ -0,0 +1,44 @@ +{ + # NOTE: the args not used in this file CAN NOT be removed! + # because haumea pass argument lazily, + # and these arguments are used in the functions like `mylib.nixosSystem`, `mylib.colmenaSystem`, etc. + inputs, + lib, + mylib, + myvars, + system, + genSpecialArgs, + ... +} @ args: let + name = "k3s-test-1-master-1"; + tags = [name]; + ssh-user = "root"; + + modules = { + nixos-modules = + (map mylib.relativeToRoot [ + # common + "secrets/nixos.nix" + "modules/nixos/server/server.nix" + "modules/nixos/server/kubevirt-hardware-configuration.nix" + # host specific + "hosts/k8s/${name}" + ]) + ++ [ + {modules.secrets.server.kubernetes.enable = true;} + ]; + home-modules = map mylib.relativeToRoot [ + "home/linux/core.nix" + ]; + }; + + systemArgs = modules // args; +in { + nixosConfigurations.${name} = mylib.nixosSystem systemArgs; + + colmena.${name} = + mylib.colmenaSystem (systemArgs // {inherit tags ssh-user;}); + + # generate proxmox image for virtual machines without desktop environment + packages.${name} = inputs.self.nixosConfigurations.${name}.config.formats.kubevirt; +} diff --git a/outputs/x86_64-linux/src/k3s-test-1-master-2.nix b/outputs/x86_64-linux/src/k3s-test-1-master-2.nix new file mode 100644 index 00000000..c659c0ea --- /dev/null +++ b/outputs/x86_64-linux/src/k3s-test-1-master-2.nix @@ -0,0 +1,41 @@ +{ + # NOTE: the args not used in this file CAN NOT be removed! + # because haumea pass argument lazily, + # and these arguments are used in the functions like `mylib.nixosSystem`, `mylib.colmenaSystem`, etc. + inputs, + lib, + mylib, + myvars, + system, + genSpecialArgs, + ... +} @ args: let + name = "k3s-test-1-master-2"; + tags = [name]; + ssh-user = "root"; + + modules = { + nixos-modules = + (map mylib.relativeToRoot [ + # common + "secrets/nixos.nix" + "modules/nixos/server/server.nix" + "modules/nixos/server/kubevirt-hardware-configuration.nix" + # host specific + "hosts/k8s/${name}" + ]) + ++ [ + {modules.secrets.server.kubernetes.enable = true;} + ]; + }; + + systemArgs = modules // args; +in { + nixosConfigurations.${name} = mylib.nixosSystem systemArgs; + + colmena.${name} = + mylib.colmenaSystem (systemArgs // {inherit tags ssh-user;}); + + # generate proxmox image for virtual machines without desktop environment + packages.${name} = inputs.self.nixosConfigurations.${name}.config.formats.kubevirt; +} diff --git a/outputs/x86_64-linux/src/k3s-test-1-master-3.nix b/outputs/x86_64-linux/src/k3s-test-1-master-3.nix new file mode 100644 index 00000000..2e6ac83f --- /dev/null +++ b/outputs/x86_64-linux/src/k3s-test-1-master-3.nix @@ -0,0 +1,41 @@ +{ + # NOTE: the args not used in this file CAN NOT be removed! + # because haumea pass argument lazily, + # and these arguments are used in the functions like `mylib.nixosSystem`, `mylib.colmenaSystem`, etc. + inputs, + lib, + mylib, + myvars, + system, + genSpecialArgs, + ... +} @ args: let + name = "k3s-test-1-master-3"; + tags = [name]; + ssh-user = "root"; + + modules = { + nixos-modules = + (map mylib.relativeToRoot [ + # common + "secrets/nixos.nix" + "modules/nixos/server/server.nix" + "modules/nixos/server/kubevirt-hardware-configuration.nix" + # host specific + "hosts/k8s/${name}" + ]) + ++ [ + {modules.secrets.server.kubernetes.enable = true;} + ]; + }; + + systemArgs = modules // args; +in { + nixosConfigurations.${name} = mylib.nixosSystem systemArgs; + + colmena.${name} = + mylib.colmenaSystem (systemArgs // {inherit tags ssh-user;}); + + # generate proxmox image for virtual machines without desktop environment + packages.${name} = inputs.self.nixosConfigurations.${name}.config.formats.kubevirt; +} diff --git a/secrets/nixos.nix b/secrets/nixos.nix index 84bada05..74243dc1 100644 --- a/secrets/nixos.nix +++ b/secrets/nixos.nix @@ -236,6 +236,12 @@ in { file = "${mysecrets}/server/k3s-prod-1-token.age"; } // high_security; + + "k3s-test-1-token" = + { + file = "${mysecrets}/server/k3s-test-1-token.age"; + } + // high_security; }; }) ]); diff --git a/utils.nu b/utils.nu index 6e7a36c9..1b95a1ea 100644 --- a/utils.nu +++ b/utils.nu @@ -53,3 +53,22 @@ export def darwin-switch [ export def darwin-rollback [] { ./result/sw/bin/darwin-rebuild --rollback } + +# ==================== Virutal Machines related ===================== + +# Build and upload a VM image +export def upload-vm [ + name: string + mode: string +] { + let target = $".#($name)" + if "debug" == $mode { + nom build $target --show-trace --verbose + } else { + nix build $target + } + + let remote = $"root@rakushun:/var/lib/caddy/fileserver/vms/kubevirt-($name).qcow2" + rsync -avz --progress --copy-links result $remote +} + diff --git a/vars/networking.nix b/vars/networking.nix index 23f08bfb..f9f7c912 100644 --- a/vars/networking.nix +++ b/vars/networking.nix @@ -101,6 +101,22 @@ iface = "ens18"; ipv4 = "192.168.5.113"; }; + + k3s-test-1-master-1 = { + # KubeVirt VM + iface = "eth1"; + ipv4 = "192.168.5.114"; + }; + k3s-test-1-master-2 = { + # KubeVirt VM + iface = "eth1"; + ipv4 = "192.168.5.115"; + }; + k3s-test-1-master-3 = { + # KubeVirt VM + iface = "eth1"; + ipv4 = "192.168.5.116"; + }; }; hostsInterface = @@ -109,10 +125,12 @@ key: val: { interfaces."${val.iface}" = { useDHCP = false; - ipv4.addresses = [{ - inherit prefixLength; - address = val.ipv4; - }]; + ipv4.addresses = [ + { + inherit prefixLength; + address = val.ipv4; + } + ]; }; } )