Files
nix-config/lib/genKubeVirtHostModule.nix
T
ryan4yin | 二花 12505b415c refactor(kubevirt): memory tuning - drop static hugepages, add zram + earlyoom (#280)
* refactor(kubevirt): drop the static 1G hugetlb pool, use THP on demand

The hosts pre-reserved 48/48/15 GiB of 1G hugepages, but the VMs only used 32/16/11 GiB, so ~52 GiB was stranded (hugetlb is exclusive and cannot be overcommitted or shared with the host / non-hugepage VMs). It also blocked scheduling (a 16 GiB hugepage VM cannot fit on a host with fewer free pages, e.g. youko with 15/24 GiB) and made the full intended VM set (idols 3x8G + k3s-test 3x16G + k3s-prod 3x4G + 3x16G = 132 GiB vs 146 GiB physical) impossible to fit.

Use transparent huge pages on demand (madvise) instead, and let the VMs use ordinary memory (instancetypes updated in k8s-gitops). This frees the stranded memory, removes the scheduling rigidity, and makes overcommit/swap usable again.

* feat(kubevirt): add earlyoom; stop overriding swappiness (zram already enabled)

zram is already provided by modules/nixos/base/zram.nix (enabled by default: 50% RAM, zstd, prio 100 + swappiness/watermark tunings). But genKubeVirtHostModule hard-coded vm.swappiness = 0, which overrode the modules mkDefault 180 and disabled swapping entirely - so the zram device was never used (Swap used: 0B).

- Remove the hard-coded vm.swappiness (let the zram modules tuning apply); the disk swapfile from disko stays as the last resort.
- Add services.earlyoom to kill the greediest process before the host thrashes / OOMs.
2026-09-21 01:39:51 +08:00

159 lines
4.8 KiB
Nix
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
{
pkgs,
hostName,
networking,
...
}:
let
inherit (networking) proxyGateway proxyGateway6 nameservers;
inherit (networking.hostsAddr.${hostName}) iface ipv4;
ipv4WithMask = "${ipv4}/24";
in
{
# supported file systems, so we can mount any removable disks with these filesystems
boot.supportedFilesystems = [
"ext4"
"btrfs"
"xfs"
#"zfs"
"ntfs"
"fat"
"vfat"
"exfat"
"nfs" # required by longhorn
];
boot.kernelModules = [
"kvm-amd"
"vfio-pci"
];
boot.extraModprobeConfig = "options kvm_amd nested=1"; # for amd cpu
boot.kernel.sysctl = {
# --- filesystem --- #
# increase the limits to avoid running out of inotify watches
"fs.inotify.max_user_watches" = 524288;
"fs.inotify.max_user_instances" = 1024;
# --- network --- #
"net.bridge.bridge-nf-call-iptables" = 1;
"net.core.somaxconn" = 32768;
# ----- IPv4 ----- #
"net.ipv4.ip_forward" = 1; # Enable forwarding
"net.ipv4.conf.all.forwarding" = 1;
"net.ipv4.neigh.default.gc_thresh1" = 4096;
"net.ipv4.neigh.default.gc_thresh2" = 6144;
"net.ipv4.neigh.default.gc_thresh3" = 8192;
"net.ipv4.neigh.default.gc_interval" = 60;
"net.ipv4.neigh.default.gc_stale_time" = 120;
# ----- IPv6 ----- #
"net.ipv6.conf.all.forwarding" = 1; # Enable forwarding
# NOTE: vm.swappiness is intentionally NOT set here; it comes from
# modules/nixos/base/zram.nix (mkDefault 180, tuned for the zram device).
# The previous hard-coded 0 disabled swapping entirely, so the zram device
# (already enabled by that module) was never used.
};
# zram itself is provided by modules/nixos/base/zram.nix (enabled by default).
# Kill the greediest process before the host starts thrashing / gets OOM-killed.
services.earlyoom = {
enable = true;
freeMemThresholdPercent = 5;
freeSwapThresholdPercent = 5;
};
environment.systemPackages = with pkgs; [
# Validate Hardware Virtualization Support via:
# virt-host-validate qemu
libvirt
kubevirt # virtctl
];
# Workaround for longhorn running on NixOS
# https://github.com/longhorn/longhorn/issues/2166
systemd.tmpfiles.rules = [
"L+ /usr/local/bin - - - - /run/current-system/sw/bin/"
];
# Longhorn uses open-iscsi to create block devices.
services.openiscsi = {
name = "iqn.2020-08.org.linux-iscsi.initiatorhost:${hostName}";
enable = true;
};
networking = {
inherit hostName;
# we use networkd instead
networkmanager.enable = false;
useDHCP = false;
};
networking.useNetworkd = true;
systemd.network.enable = true;
# Linux bridge for the VMs' secondary network. It replaces the previous OVS
# bridge: the `bridge` CNI plugin attaches the VM taps to br0, and the host
# keeps its own address on br0 (the physical NIC is just a bridge port).
systemd.network.netdevs."10-br0" = {
netdevConfig = {
Name = "br0";
Kind = "bridge";
};
};
# systemd.services."systemd-networkd".environment.SYSTEMD_LOG_LEVEL = "debug";
# Set the host's address on the bridge interface instead of the physical interface!
systemd.network.networks = {
"10-br0" = {
matchConfig.Name = [ "br0" ];
networkConfig = {
Address = [ ipv4WithMask ];
# DNS = nameservers;
DNS = [ proxyGateway ];
DHCP = "ipv6"; # enable DHCPv6 only, so we can get a GUA.
IPv6AcceptRA = true; # for Stateless IPv6 Autoconfiguraton (SLAAC)
LinkLocalAddressing = "ipv6";
};
routes = [
{
Destination = "0.0.0.0/0";
Gateway = proxyGateway;
}
{
Destination = "::/0";
Gateway = proxyGateway6;
GatewayOnLink = true; # it's a gateway on local link.
}
];
linkConfig.RequiredForOnline = "routable";
};
"20-${iface}" = {
matchConfig.Name = [ iface ];
networkConfig = {
LinkLocalAddressing = "no";
# attach the physical NIC to the Linux bridge br0
Bridge = "br0";
};
linkConfig.RequiredForOnline = "no";
};
};
# systemd-journal - cap disk usage, but keep plenty of history on the big
# persistent disk (/var/log is preserved to /persistent).
# https://www.freedesktop.org/software/systemd/man/latest/journald.conf.html
services.journald.settings.Journal = {
SystemMaxUse = "10G";
RuntimeMaxUse = "256M";
};
# This value determines the NixOS release from which the default
# settings for stateful data, like file locations and database versions
# on your system were taken. It‘s perfectly fine and recommended to leave
# this value at the release version of the first install of this system.
# Before changing this value read the documentation for this option
# (e.g. man configuration.nix or on https://nixos.org/nixos/options.html).
system.stateVersion = "24.11"; # Did you read the comment?
}