From 12505b415cb9e10f71bfe8057e79b59e782ae968 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?ryan4yin=20=7C=20=E4=BA=8C=E8=8A=B1?= Date: Mon, 21 Sep 2026 01:39:51 +0800 Subject: [PATCH] refactor(kubevirt): memory tuning - drop static hugepages, add zram + earlyoom (#280) * refactor(kubevirt): drop the static 1G hugetlb pool, use THP on demand The hosts pre-reserved 48/48/15 GiB of 1G hugepages, but the VMs only used 32/16/11 GiB, so ~52 GiB was stranded (hugetlb is exclusive and cannot be overcommitted or shared with the host / non-hugepage VMs). It also blocked scheduling (a 16 GiB hugepage VM cannot fit on a host with fewer free pages, e.g. youko with 15/24 GiB) and made the full intended VM set (idols 3x8G + k3s-test 3x16G + k3s-prod 3x4G + 3x16G = 132 GiB vs 146 GiB physical) impossible to fit. Use transparent huge pages on demand (madvise) instead, and let the VMs use ordinary memory (instancetypes updated in k8s-gitops). This frees the stranded memory, removes the scheduling rigidity, and makes overcommit/swap usable again. * feat(kubevirt): add earlyoom; stop overriding swappiness (zram already enabled) zram is already provided by modules/nixos/base/zram.nix (enabled by default: 50% RAM, zstd, prio 100 + swappiness/watermark tunings). But genKubeVirtHostModule hard-coded vm.swappiness = 0, which overrode the modules mkDefault 180 and disabled swapping entirely - so the zram device was never used (Swap used: 0B). - Remove the hard-coded vm.swappiness (let the zram modules tuning apply); the disk swapfile from disko stays as the last resort. - Add services.earlyoom to kill the greediest process before the host thrashes / OOMs. --- hosts/k8s/kubevirt-shoryu/default.nix | 15 +++++---------- hosts/k8s/kubevirt-shushou/default.nix | 15 +++++---------- hosts/k8s/kubevirt-youko/default.nix | 14 ++++---------- lib/genKubeVirtHostModule.nix | 14 ++++++++++++-- 4 files changed, 26 insertions(+), 32 deletions(-) diff --git a/hosts/k8s/kubevirt-shoryu/default.nix b/hosts/k8s/kubevirt-shoryu/default.nix index 5b8cfc36..7043f500 100644 --- a/hosts/k8s/kubevirt-shoryu/default.nix +++ b/hosts/k8s/kubevirt-shoryu/default.nix @@ -57,15 +57,10 @@ in modules.btrbk.enable = true; boot.kernelParams = [ - # disable transparent hugepage(allocate hugepages dynamically) - "transparent_hugepage=never" - - # https://kubevirt.io/user-guide/compute/hugepages/ - # - # pre-allocate hugepages manually(for kubevirt guest vms) - # NOTE: the hugepages allocated here can not be used for other purposes! - # so we should left some memory for the host OS and other vms that don't use hugepages - "hugepagesz=1G" - "hugepages=48" # use 75% of the total memory for hugepages + # Use transparent huge pages on demand (madvise) instead of a fixed 1G hugetlb + # pool. The static pool cannot be overcommitted or shared with the host / + # other VMs, which stranded memory and blocked scheduling; the VMs now use + # ordinary memory (see the instancetypes in k8s-gitops). + "transparent_hugepage=madvise" ]; } diff --git a/hosts/k8s/kubevirt-shushou/default.nix b/hosts/k8s/kubevirt-shushou/default.nix index b160612b..e6754772 100644 --- a/hosts/k8s/kubevirt-shushou/default.nix +++ b/hosts/k8s/kubevirt-shushou/default.nix @@ -54,15 +54,10 @@ in modules.btrbk.enable = true; boot.kernelParams = [ - # disable transparent hugepage(allocate hugepages dynamically) - "transparent_hugepage=never" - - # https://kubevirt.io/user-guide/compute/hugepages/ - # - # pre-allocate hugepages manually(for kubevirt guest vms) - # NOTE: the hugepages allocated here can not be used for other purposes! - # so we should left some memory for the host OS and other vms that don't use hugepages - "hugepagesz=1G" - "hugepages=48" # use 75% of the total memory for hugepages + # Use transparent huge pages on demand (madvise) instead of a fixed 1G hugetlb + # pool. The static pool cannot be overcommitted or shared with the host / + # other VMs, which stranded memory and blocked scheduling; the VMs now use + # ordinary memory (see the instancetypes in k8s-gitops). + "transparent_hugepage=madvise" ]; } diff --git a/hosts/k8s/kubevirt-youko/default.nix b/hosts/k8s/kubevirt-youko/default.nix index 6911116c..c77bd7fe 100644 --- a/hosts/k8s/kubevirt-youko/default.nix +++ b/hosts/k8s/kubevirt-youko/default.nix @@ -54,16 +54,10 @@ in modules.btrbk.enable = true; boot.kernelParams = [ - # disable transparent hugepage(allocate hugepages dynamically) - "transparent_hugepage=never" - - # https://kubevirt.io/user-guide/compute/hugepages/ - # - # pre-allocate hugepages manually(for kubevirt guest vms) - # NOTE: the hugepages allocated here can not be used for other purposes! - # so we should left some memory for the host OS and other vms that don't use hugepages - "hugepagesz=1G" - "hugepages=15" # use 15/24 of the total memory for hugepages + # Use transparent huge pages on demand (madvise) instead of a fixed 1G hugetlb + # pool (cannot be overcommitted/shared; it stranded memory and blocked + # scheduling). The VMs now use ordinary memory. + "transparent_hugepage=madvise" # https://kubevirt.io/user-guide/compute/host-devices/ # diff --git a/lib/genKubeVirtHostModule.nix b/lib/genKubeVirtHostModule.nix index ffc85a06..d1704bbe 100644 --- a/lib/genKubeVirtHostModule.nix +++ b/lib/genKubeVirtHostModule.nix @@ -50,8 +50,18 @@ in # ----- IPv6 ----- # "net.ipv6.conf.all.forwarding" = 1; # Enable forwarding - # --- memory --- # - "vm.swappiness" = 0; # don't swap unless absolutely necessary + # NOTE: vm.swappiness is intentionally NOT set here; it comes from + # modules/nixos/base/zram.nix (mkDefault 180, tuned for the zram device). + # The previous hard-coded 0 disabled swapping entirely, so the zram device + # (already enabled by that module) was never used. + }; + + # zram itself is provided by modules/nixos/base/zram.nix (enabled by default). + # Kill the greediest process before the host starts thrashing / gets OOM-killed. + services.earlyoom = { + enable = true; + freeMemThresholdPercent = 5; + freeSwapThresholdPercent = 5; }; environment.systemPackages = with pkgs; [