mirror of
https://github.com/ryan4yin/nix-config.git
synced 2026-11-18 05:58:17 +01:00
refactor(kubevirt): memory tuning - drop static hugepages, add zram + earlyoom (#280)
* refactor(kubevirt): drop the static 1G hugetlb pool, use THP on demand The hosts pre-reserved 48/48/15 GiB of 1G hugepages, but the VMs only used 32/16/11 GiB, so ~52 GiB was stranded (hugetlb is exclusive and cannot be overcommitted or shared with the host / non-hugepage VMs). It also blocked scheduling (a 16 GiB hugepage VM cannot fit on a host with fewer free pages, e.g. youko with 15/24 GiB) and made the full intended VM set (idols 3x8G + k3s-test 3x16G + k3s-prod 3x4G + 3x16G = 132 GiB vs 146 GiB physical) impossible to fit. Use transparent huge pages on demand (madvise) instead, and let the VMs use ordinary memory (instancetypes updated in k8s-gitops). This frees the stranded memory, removes the scheduling rigidity, and makes overcommit/swap usable again. * feat(kubevirt): add earlyoom; stop overriding swappiness (zram already enabled) zram is already provided by modules/nixos/base/zram.nix (enabled by default: 50% RAM, zstd, prio 100 + swappiness/watermark tunings). But genKubeVirtHostModule hard-coded vm.swappiness = 0, which overrode the modules mkDefault 180 and disabled swapping entirely - so the zram device was never used (Swap used: 0B). - Remove the hard-coded vm.swappiness (let the zram modules tuning apply); the disk swapfile from disko stays as the last resort. - Add services.earlyoom to kill the greediest process before the host thrashes / OOMs.
This commit is contained in:
1 parent
64a81f16ec
commit
12505b415c
4 files changed
+26
-32
No files matched your search
@@ -57,15 +57,10 @@ in
|
||||
modules.btrbk.enable = true;
|
||||
|
||||
boot.kernelParams = [
|
||||
# disable transparent hugepage(allocate hugepages dynamically)
|
||||
"transparent_hugepage=never"
|
||||
|
||||
# https://kubevirt.io/user-guide/compute/hugepages/
|
||||
#
|
||||
# pre-allocate hugepages manually(for kubevirt guest vms)
|
||||
# NOTE: the hugepages allocated here can not be used for other purposes!
|
||||
# so we should left some memory for the host OS and other vms that don't use hugepages
|
||||
"hugepagesz=1G"
|
||||
"hugepages=48" # use 75% of the total memory for hugepages
|
||||
# Use transparent huge pages on demand (madvise) instead of a fixed 1G hugetlb
|
||||
# pool. The static pool cannot be overcommitted or shared with the host /
|
||||
# other VMs, which stranded memory and blocked scheduling; the VMs now use
|
||||
# ordinary memory (see the instancetypes in k8s-gitops).
|
||||
"transparent_hugepage=madvise"
|
||||
];
|
||||
}
|
||||
@@ -54,15 +54,10 @@ in
|
||||
modules.btrbk.enable = true;
|
||||
|
||||
boot.kernelParams = [
|
||||
# disable transparent hugepage(allocate hugepages dynamically)
|
||||
"transparent_hugepage=never"
|
||||
|
||||
# https://kubevirt.io/user-guide/compute/hugepages/
|
||||
#
|
||||
# pre-allocate hugepages manually(for kubevirt guest vms)
|
||||
# NOTE: the hugepages allocated here can not be used for other purposes!
|
||||
# so we should left some memory for the host OS and other vms that don't use hugepages
|
||||
"hugepagesz=1G"
|
||||
"hugepages=48" # use 75% of the total memory for hugepages
|
||||
# Use transparent huge pages on demand (madvise) instead of a fixed 1G hugetlb
|
||||
# pool. The static pool cannot be overcommitted or shared with the host /
|
||||
# other VMs, which stranded memory and blocked scheduling; the VMs now use
|
||||
# ordinary memory (see the instancetypes in k8s-gitops).
|
||||
"transparent_hugepage=madvise"
|
||||
];
|
||||
}
|
||||
@@ -54,16 +54,10 @@ in
|
||||
modules.btrbk.enable = true;
|
||||
|
||||
boot.kernelParams = [
|
||||
# disable transparent hugepage(allocate hugepages dynamically)
|
||||
"transparent_hugepage=never"
|
||||
|
||||
# https://kubevirt.io/user-guide/compute/hugepages/
|
||||
#
|
||||
# pre-allocate hugepages manually(for kubevirt guest vms)
|
||||
# NOTE: the hugepages allocated here can not be used for other purposes!
|
||||
# so we should left some memory for the host OS and other vms that don't use hugepages
|
||||
"hugepagesz=1G"
|
||||
"hugepages=15" # use 15/24 of the total memory for hugepages
|
||||
# Use transparent huge pages on demand (madvise) instead of a fixed 1G hugetlb
|
||||
# pool (cannot be overcommitted/shared; it stranded memory and blocked
|
||||
# scheduling). The VMs now use ordinary memory.
|
||||
"transparent_hugepage=madvise"
|
||||
|
||||
# https://kubevirt.io/user-guide/compute/host-devices/
|
||||
#
|
||||
|
||||
@@ -50,8 +50,18 @@ in
|
||||
# ----- IPv6 ----- #
|
||||
"net.ipv6.conf.all.forwarding" = 1; # Enable forwarding
|
||||
|
||||
# --- memory --- #
|
||||
"vm.swappiness" = 0; # don't swap unless absolutely necessary
|
||||
# NOTE: vm.swappiness is intentionally NOT set here; it comes from
|
||||
# modules/nixos/base/zram.nix (mkDefault 180, tuned for the zram device).
|
||||
# The previous hard-coded 0 disabled swapping entirely, so the zram device
|
||||
# (already enabled by that module) was never used.
|
||||
};
|
||||
|
||||
# zram itself is provided by modules/nixos/base/zram.nix (enabled by default).
|
||||
# Kill the greediest process before the host starts thrashing / gets OOM-killed.
|
||||
services.earlyoom = {
|
||||
enable = true;
|
||||
freeMemThresholdPercent = 5;
|
||||
freeSwapThresholdPercent = 5;
|
||||
};
|
||||
|
||||
environment.systemPackages = with pkgs; [
|
||||
|
||||
Reference in new issue
Block a user