diff --git a/hosts/k8s/kubevirt-shoryu/default.nix b/hosts/k8s/kubevirt-shoryu/default.nix index 55fe587d..93edf91c 100644 --- a/hosts/k8s/kubevirt-shoryu/default.nix +++ b/hosts/k8s/kubevirt-shoryu/default.nix @@ -1,5 +1,5 @@ { - config, + lib, pkgs, mylib, myvars, @@ -22,6 +22,13 @@ # use my own domain & kube-vip's virtual IP for the API server # so that the API server can always be accessed even if some nodes are down masterHost = "kubevirt-cluster-1.writefor.fun"; + kubeletExtraArgs = [ + "--cpu-manager-policy=static" + # https://kubernetes.io/docs/tasks/administer-cluster/reserve-compute-resources/ + # we have to reserve some resources for for system daemons running as pods + # when cpu-manager's static policy is enabled + "--system-reserved=cpu=1,memory=1Gi,ephemeral-storage=2Gi" + ]; nodeLabels = [ "node-purpose=kubevirt" ]; diff --git a/lib/genK3sServerModule.nix b/lib/genK3sServerModule.nix index 22ed2c12..a4dc40dd 100644 --- a/lib/genK3sServerModule.nix +++ b/lib/genK3sServerModule.nix @@ -10,11 +10,13 @@ # this can be a domain name or an IP address(such as kube-vip's virtual IP) masterHost, clusterInit ? false, + kubeletExtraArgs ? [], nodeLabels ? [], nodeTaints ? [], disableFlannel ? true, ... }: let + lib = pkgs.lib; package = pkgs.k3s_1_29; in { environment.systemPackages = with pkgs; [ @@ -59,9 +61,10 @@ in { ] ++ (map (label: "--node-label=${label}") nodeLabels) ++ (map (taint: "--node-taint=${taint}") nodeTaints) - ++ (pkgs.lib.optionals disableFlannel ["--flannel-backend=none"]); + ++ (map (arg: "--kubelet-arg=${arg}") kubeletExtraArgs) + ++ (lib.optionals disableFlannel ["--flannel-backend=none"]); in - pkgs.lib.concatStringsSep " " flagList; + lib.concatStringsSep " " flagList; }; # create symlinks to link k3s's cni directory to the one used by almost all CNI plugins