From cf2435e8545232f52e9dd9959ea2987631df20ea Mon Sep 17 00:00:00 2001 From: Ryan Yin Date: Mon, 21 Sep 2026 16:57:42 +0800 Subject: [PATCH 1/3] chore(hosts): retire the k3s cluster on the physical hosts The physical hosts no longer need k3s or KubeVirt: aquamarine runs natively, the k3s-test masters run as microVMs and kana/ruby under libvirt. So drop the host cluster and split the shared parts out of the KubeVirt host module: - lib/genVmHostModule.nix: the host bits the VMs still need (br0 + the physical NIC as a bridge port, KVM, sysctls, earlyoom, libvirt, journald, stateVersion) - the three hosts use it and drop genK3sServerModule; also drop the longhorn workarounds (openiscsi + the /usr/local/bin tmpfile) and the virtctl package - remove lib/genKubeVirtHostModule.nix --- hosts/k8s/kubevirt-shoryu/default.nix | 32 +----------------- hosts/k8s/kubevirt-shushou/default.nix | 30 +---------------- hosts/k8s/kubevirt-youko/default.nix | 30 +---------------- lib/default.nix | 2 +- ...VirtHostModule.nix => genVmHostModule.nix} | 33 ++++--------------- 5 files changed, 11 insertions(+), 116 deletions(-) rename lib/{genKubeVirtHostModule.nix => genVmHostModule.nix} (78%) diff --git a/hosts/k8s/kubevirt-shoryu/default.nix b/hosts/k8s/kubevirt-shoryu/default.nix index 7043f500..360b987d 100644 --- a/hosts/k8s/kubevirt-shoryu/default.nix +++ b/hosts/k8s/kubevirt-shoryu/default.nix @@ -10,39 +10,10 @@ let # MoreFine - S500Plus hostName = "kubevirt-shoryu"; # Define your hostname. - coreModule = mylib.genKubeVirtHostModule { + coreModule = mylib.genVmHostModule { inherit pkgs hostName; inherit (myvars) networking; }; - k3sModule = mylib.genK3sServerModule { - inherit pkgs; - kubeconfigFile = "/home/${myvars.username}/.kube/config"; - tokenFile = "/persistent/kubevirt-k3s-token"; - # the first node in the cluster should be the one to initialize the cluster - clusterInit = true; - # use my own domain & kube-vip's virtual IP for the API server - # so that the API server can always be accessed even if some nodes are down - masterHost = "kubevirt-cluster-1.writefor.fun"; - kubeletExtraArgs = [ - "--cpu-manager-policy=static" - # https://kubernetes.io/docs/tasks/administer-cluster/reserve-compute-resources/ - # we have to reserve some resources for for system daemons running as pods or system services - # when cpu-manager's static policy is enabled - # the memory we reserved here is also for the kernel, since kernel's memory is not accounted in pods - "--system-reserved=cpu=1,memory=2Gi,ephemeral-storage=2Gi" - ]; - k3sExtraArgs = [ - # IPv4 Private CIDR(full) - 172.16.0.0/12 - # IPv4 Pod CIDR(full) - fdfd:cafe:00:0000::/64 ~ fdfd:cafe:00:7fff::/64 - # IPv4 Service CIDR(full) - fdfd:cafe:00:8000::/64 ~ fdfd:cafe:00:ffff::/64 - # "--cluster-cidr=172.16.0.0/16,fdfd:cafe:00:0001::/64" - # "--service-cidr=172.17.0.0/16,fdfd:cafe:00:8001::/112" - ]; - nodeLabels = [ - "node-purpose=kubevirt" - ]; - # flannel is disabled; Cilium provides the pod network. - }; in { imports = (mylib.scanPaths ./.) ++ [ @@ -51,7 +22,6 @@ in ./hardware-configuration.nix ./preservation.nix coreModule - k3sModule ]; modules.btrbk.enable = true; diff --git a/hosts/k8s/kubevirt-shushou/default.nix b/hosts/k8s/kubevirt-shushou/default.nix index e6754772..90bb7a17 100644 --- a/hosts/k8s/kubevirt-shushou/default.nix +++ b/hosts/k8s/kubevirt-shushou/default.nix @@ -9,37 +9,10 @@ let hostName = "kubevirt-shushou"; # Define your hostname. - coreModule = mylib.genKubeVirtHostModule { + coreModule = mylib.genVmHostModule { inherit pkgs hostName; inherit (myvars) networking; }; - k3sModule = mylib.genK3sServerModule { - inherit pkgs; - kubeconfigFile = "/home/${myvars.username}/.kube/config"; - tokenFile = "/persistent/kubevirt-k3s-token"; - # use my own domain & kube-vip's virtual IP for the API server - # so that the API server can always be accessed even if some nodes are down - masterHost = "kubevirt-cluster-1.writefor.fun"; - kubeletExtraArgs = [ - "--cpu-manager-policy=static" - # https://kubernetes.io/docs/tasks/administer-cluster/reserve-compute-resources/ - # we have to reserve some resources for for system daemons running as pods or system services - # when cpu-manager's static policy is enabled - # the memory we reserved here is also for the kernel, since kernel's memory is not accounted in pods - "--system-reserved=cpu=1,memory=2Gi,ephemeral-storage=2Gi" - ]; - k3sExtraArgs = [ - # IPv4 Private CIDR(full) - 172.16.0.0/12 - # IPv4 Pod CIDR(full) - fdfd:cafe:00:0000::/64 ~ fdfd:cafe:00:7fff::/64 - # IPv4 Service CIDR(full) - fdfd:cafe:00:8000::/64 ~ fdfd:cafe:00:ffff::/64 - # "--cluster-cidr=172.16.0.0/16,fdfd:cafe:00:0001::/64" - # "--service-cidr=172.17.0.0/16,fdfd:cafe:00:8001::/112" - ]; - nodeLabels = [ - "node-purpose=kubevirt" - ]; - # flannel is disabled; Cilium provides the pod network. - }; in { imports = (mylib.scanPaths ./.) ++ [ @@ -48,7 +21,6 @@ in ../kubevirt-shoryu/hardware-configuration.nix ../kubevirt-shoryu/preservation.nix coreModule - k3sModule ]; modules.btrbk.enable = true; diff --git a/hosts/k8s/kubevirt-youko/default.nix b/hosts/k8s/kubevirt-youko/default.nix index c77bd7fe..54d3d127 100644 --- a/hosts/k8s/kubevirt-youko/default.nix +++ b/hosts/k8s/kubevirt-youko/default.nix @@ -9,37 +9,10 @@ let hostName = "kubevirt-youko"; # Define your hostname. - coreModule = mylib.genKubeVirtHostModule { + coreModule = mylib.genVmHostModule { inherit pkgs hostName; inherit (myvars) networking; }; - k3sModule = mylib.genK3sServerModule { - inherit pkgs; - kubeconfigFile = "/home/${myvars.username}/.kube/config"; - tokenFile = "/persistent/kubevirt-k3s-token"; - # use my own domain & kube-vip's virtual IP for the API server - # so that the API server can always be accessed even if some nodes are down - masterHost = "kubevirt-cluster-1.writefor.fun"; - kubeletExtraArgs = [ - "--cpu-manager-policy=static" - # https://kubernetes.io/docs/tasks/administer-cluster/reserve-compute-resources/ - # we have to reserve some resources for for system daemons running as pods or system services - # when cpu-manager's static policy is enabled - # the memory we reserved here is also for the kernel, since kernel's memory is not accounted in pods - "--system-reserved=cpu=1,memory=2Gi,ephemeral-storage=2Gi" - ]; - k3sExtraArgs = [ - # IPv4 Private CIDR(full) - 172.16.0.0/12 - # IPv4 Pod CIDR(full) - fdfd:cafe:00:0000::/64 ~ fdfd:cafe:00:7fff::/64 - # IPv4 Service CIDR(full) - fdfd:cafe:00:8000::/64 ~ fdfd:cafe:00:ffff::/64 - # "--cluster-cidr=172.16.0.0/16,fdfd:cafe:00:0001::/64" - # "--service-cidr=172.17.0.0/16,fdfd:cafe:00:8001::/112" - ]; - nodeLabels = [ - "node-purpose=kubevirt" - ]; - # flannel is disabled; Cilium provides the pod network. - }; in { imports = (mylib.scanPaths ./.) ++ [ @@ -48,7 +21,6 @@ in ../kubevirt-shoryu/hardware-configuration.nix ../kubevirt-shoryu/preservation.nix coreModule - k3sModule ]; modules.btrbk.enable = true; diff --git a/lib/default.nix b/lib/default.nix index 3f762523..f72ffb6d 100644 --- a/lib/default.nix +++ b/lib/default.nix @@ -8,7 +8,7 @@ genK3sServerModule = import ./genK3sServerModule.nix; genK3sAgentModule = import ./genK3sAgentModule.nix; - genKubeVirtHostModule = import ./genKubeVirtHostModule.nix; + genVmHostModule = import ./genVmHostModule.nix; genKubeVirtGuestModule = import ./genKubeVirtGuestModule.nix; genMicrovmGuestModule = import ./genMicrovmGuestModule.nix; diff --git a/lib/genKubeVirtHostModule.nix b/lib/genVmHostModule.nix similarity index 78% rename from lib/genKubeVirtHostModule.nix rename to lib/genVmHostModule.nix index 5c41a79e..2b2c0b28 100644 --- a/lib/genKubeVirtHostModule.nix +++ b/lib/genVmHostModule.nix @@ -5,7 +5,7 @@ ... }: let - inherit (networking) proxyGateway proxyGateway6 nameservers; + inherit (networking) proxyGateway proxyGateway6; inherit (networking.hostsAddr.${hostName}) iface ipv4; ipv4WithMask = "${ipv4}/24"; in @@ -20,13 +20,11 @@ in "fat" "vfat" "exfat" - "nfs" # required by longhorn + "nfs" ]; - boot.kernelModules = [ - "kvm-amd" - "vfio-pci" - ]; + # KVM for the microVMs and the libvirt domains. + boot.kernelModules = [ "kvm-amd" ]; boot.extraModprobeConfig = "options kvm_amd nested=1"; # for amd cpu boot.kernel.sysctl = { @@ -52,33 +50,18 @@ in # NOTE: vm.swappiness is intentionally NOT set here; it comes from # modules/nixos/base/zram.nix (mkDefault 180, tuned for the zram device). - # The previous hard-coded 0 disabled swapping entirely, so the zram device - # (already enabled by that module) was never used. }; # zram itself is provided by modules/nixos/base/zram.nix (enabled by default). # Kill the greediest process before the host starts thrashing / gets OOM-killed. - # (defaults: trigger at 10% free mem / swap, kill hard at half of that.) services.earlyoom.enable = true; environment.systemPackages = with pkgs; [ # Validate Hardware Virtualization Support via: # virt-host-validate qemu libvirt - kubevirt # virtctl ]; - # Workaround for longhorn running on NixOS - # https://github.com/longhorn/longhorn/issues/2166 - systemd.tmpfiles.rules = [ - "L+ /usr/local/bin - - - - /run/current-system/sw/bin/" - ]; - # Longhorn uses open-iscsi to create block devices. - services.openiscsi = { - name = "iqn.2020-08.org.linux-iscsi.initiatorhost:${hostName}"; - enable = true; - }; - networking = { inherit hostName; @@ -89,9 +72,9 @@ in networking.useNetworkd = true; systemd.network.enable = true; - # Linux bridge for the VMs' secondary network. It replaces the previous OVS - # bridge: the `bridge` CNI plugin attaches the VM taps to br0, and the host - # keeps its own address on br0 (the physical NIC is just a bridge port). + # Linux bridge for the VMs: the microVM taps and the libvirt domains attach to + # br0, and the host keeps its own address on it (the physical NIC is just a + # bridge port). systemd.network.netdevs."10-br0" = { netdevConfig = { Name = "br0"; @@ -99,8 +82,6 @@ in }; }; - # systemd.services."systemd-networkd".environment.SYSTEMD_LOG_LEVEL = "debug"; - # Set the host's address on the bridge interface instead of the physical interface! systemd.network.networks = { "10-br0" = { From 116f2dc11422c693ed35016911c96cffc5d3d27f Mon Sep 17 00:00:00 2001 From: Ryan Yin Date: Mon, 21 Sep 2026 16:58:24 +0800 Subject: [PATCH 2/3] chore(gitignore): stop ignoring the repository's own lib/ The Python section of the .gitignore template has a bare 'lib/' (and 'lib64/'), which also matched the Nix helper modules in lib/. The '!lib/*.nix' line added earlier could not help, since a file cannot be re-included while its parent directory is excluded. --- .gitignore | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.gitignore b/.gitignore index 0083f18e..5a3188ec 100644 --- a/.gitignore +++ b/.gitignore @@ -29,8 +29,6 @@ dist/ downloads/ eggs/ .eggs/ -lib/ -lib64/ parts/ sdist/ var/ @@ -229,6 +227,3 @@ __marimo__/ # Streamlit .streamlit/secrets.toml - -# the library modules under lib/ must stay tracked -!lib/*.nix From 8d848a878a47c67fd0b91ae51ed4dfb1b64369a3 Mon Sep 17 00:00:00 2001 From: Ryan Yin Date: Mon, 21 Sep 2026 16:59:42 +0800 Subject: [PATCH 3/3] chore(gitignore): trim the Python section to what a script repo needs This is not a Python package, so drop the packaging/build/pyinstaller/ framework/type-checker/pyenv/poetry/pdm/pixi/celery/rabbitmq rules and the unused Marimo and Streamlit sections. Keep the bits occasional Python scripts do need: byte-compiled files, virtualenvs, local secrets and the common tool caches. --- .gitignore | 219 +++-------------------------------------------------- 1 file changed, 12 insertions(+), 207 deletions(-) diff --git a/.gitignore b/.gitignore index 5a3188ec..c1e8b617 100644 --- a/.gitignore +++ b/.gitignore @@ -11,219 +11,24 @@ core* !core.nix !coredns* -# =============== Python.gitignore =========================== +# =============== Python =========================== -# Byte-compiled / optimized / DLL files +# Byte-compiled / optimized files __pycache__/ -*.py[codz] +*.py[cod] *$py.class -# C extensions -*.so - -# Distribution / packaging -.Python -build/ -develop-eggs/ -dist/ -downloads/ -eggs/ -.eggs/ -parts/ -sdist/ -var/ -wheels/ -share/python-wheels/ -*.egg-info/ -.installed.cfg -*.egg -MANIFEST - -# PyInstaller -# Usually these files are written by a python script from a template -# before PyInstaller builds the exe, so as to inject date/other infos into it. -*.manifest -*.spec - -# Installer logs -pip-log.txt -pip-delete-this-directory.txt - -# Unit test / coverage reports -htmlcov/ -.tox/ -.nox/ -.coverage -.coverage.* -.cache -nosetests.xml -coverage.xml -*.cover -*.py.cover -.hypothesis/ -.pytest_cache/ -cover/ - -# Translations -*.mo -*.pot - -# Django stuff: -*.log -local_settings.py -db.sqlite3 -db.sqlite3-journal - -# Flask stuff: -instance/ -.webassets-cache - -# Scrapy stuff: -.scrapy - -# Sphinx documentation -docs/_build/ - -# PyBuilder -.pybuilder/ -target/ - -# Jupyter Notebook -.ipynb_checkpoints - -# IPython -profile_default/ -ipython_config.py - -# pyenv -# For a library or package, you might want to ignore these files since the code is -# intended to run in multiple environments; otherwise, check them in: -# .python-version - -# pipenv -# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. -# However, in case of collaboration, if having platform-specific dependencies or dependencies -# having no cross-platform support, pipenv may install dependencies that don't work, or not -# install all needed dependencies. -# Pipfile.lock - -# UV -# Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control. -# This is especially recommended for binary packages to ensure reproducibility, and is more -# commonly ignored for libraries. -# uv.lock - -# poetry -# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. -# This is especially recommended for binary packages to ensure reproducibility, and is more -# commonly ignored for libraries. -# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control -# poetry.lock -# poetry.toml - -# pdm -# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. -# pdm recommends including project-wide configuration in pdm.toml, but excluding .pdm-python. -# https://pdm-project.org/en/latest/usage/project/#working-with-version-control -# pdm.lock -# pdm.toml -.pdm-python -.pdm-build/ - -# pixi -# Similar to Pipfile.lock, it is generally recommended to include pixi.lock in version control. -# pixi.lock -# Pixi creates a virtual environment in the .pixi directory, just like venv module creates one -# in the .venv directory. It is recommended not to include this directory in version control. -.pixi - -# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm -__pypackages__/ - -# Celery stuff -celerybeat-schedule -celerybeat.pid - -# Redis -*.rdb -*.aof -*.pid - -# RabbitMQ -mnesia/ -rabbitmq/ -rabbitmq-data/ - -# ActiveMQ -activemq-data/ - -# SageMath parsed files -*.sage.py - -# Environments -.env -.envrc -.venv -env/ +# virtualenvs +.venv/ venv/ -ENV/ -env.bak/ -venv.bak/ +env/ -# Spyder project settings -.spyderproject -.spyproject +# local secrets +.env -# Rope project settings -.ropeproject - -# mkdocs documentation -/site - -# mypy +# tooling caches +.pytest_cache/ .mypy_cache/ -.dmypy.json -dmypy.json - -# Pyre type checker -.pyre/ - -# pytype static type analyzer -.pytype/ - -# Cython debug symbols -cython_debug/ - -# PyCharm -# JetBrains specific template is maintained in a separate JetBrains.gitignore that can -# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore -# and can be added to the global gitignore or merged into this file. For a more nuclear -# option (not recommended) you can uncomment the following to ignore the entire idea folder. -# .idea/ - -# Abstra -# Abstra is an AI-powered process automation framework. -# Ignore directories containing user credentials, local state, and settings. -# Learn more at https://abstra.io/docs -.abstra/ - -# Visual Studio Code -# Visual Studio Code specific template is maintained in a separate VisualStudioCode.gitignore -# that can be found at https://github.com/github/gitignore/blob/main/Global/VisualStudioCode.gitignore -# and can be added to the global gitignore or merged into this file. However, if you prefer, -# you could uncomment the following to ignore the entire vscode folder -# .vscode/ - -# Ruff stuff: .ruff_cache/ - -# PyPI configuration file -.pypirc - -# Marimo -marimo/_static/ -marimo/_lsp/ -__marimo__/ - -# Streamlit -.streamlit/secrets.toml +.coverage +htmlcov/