mirror of
https://github.com/ryan4yin/nix-config.git
synced 2026-11-13 12:13:36 +01:00
chore(hosts): retire the k3s cluster on the physical hosts
The physical hosts no longer need k3s or KubeVirt: aquamarine runs natively, the k3s-test masters run as microVMs and kana/ruby under libvirt. So drop the host cluster and split the shared parts out of the KubeVirt host module: - lib/genVmHostModule.nix: the host bits the VMs still need (br0 + the physical NIC as a bridge port, KVM, sysctls, earlyoom, libvirt, journald, stateVersion) - the three hosts use it and drop genK3sServerModule; also drop the longhorn workarounds (openiscsi + the /usr/local/bin tmpfile) and the virtctl package - remove lib/genKubeVirtHostModule.nix
This commit is contained in:
Ryan Yin committed
1 parent
6e551770ac
commit
cf2435e854
5 files changed
+11
-116
No files matched your search
@@ -10,39 +10,10 @@ let
|
||||
# MoreFine - S500Plus
|
||||
hostName = "kubevirt-shoryu"; # Define your hostname.
|
||||
|
||||
coreModule = mylib.genKubeVirtHostModule {
|
||||
coreModule = mylib.genVmHostModule {
|
||||
inherit pkgs hostName;
|
||||
inherit (myvars) networking;
|
||||
};
|
||||
k3sModule = mylib.genK3sServerModule {
|
||||
inherit pkgs;
|
||||
kubeconfigFile = "/home/${myvars.username}/.kube/config";
|
||||
tokenFile = "/persistent/kubevirt-k3s-token";
|
||||
# the first node in the cluster should be the one to initialize the cluster
|
||||
clusterInit = true;
|
||||
# use my own domain & kube-vip's virtual IP for the API server
|
||||
# so that the API server can always be accessed even if some nodes are down
|
||||
masterHost = "kubevirt-cluster-1.writefor.fun";
|
||||
kubeletExtraArgs = [
|
||||
"--cpu-manager-policy=static"
|
||||
# https://kubernetes.io/docs/tasks/administer-cluster/reserve-compute-resources/
|
||||
# we have to reserve some resources for for system daemons running as pods or system services
|
||||
# when cpu-manager's static policy is enabled
|
||||
# the memory we reserved here is also for the kernel, since kernel's memory is not accounted in pods
|
||||
"--system-reserved=cpu=1,memory=2Gi,ephemeral-storage=2Gi"
|
||||
];
|
||||
k3sExtraArgs = [
|
||||
# IPv4 Private CIDR(full) - 172.16.0.0/12
|
||||
# IPv4 Pod CIDR(full) - fdfd:cafe:00:0000::/64 ~ fdfd:cafe:00:7fff::/64
|
||||
# IPv4 Service CIDR(full) - fdfd:cafe:00:8000::/64 ~ fdfd:cafe:00:ffff::/64
|
||||
# "--cluster-cidr=172.16.0.0/16,fdfd:cafe:00:0001::/64"
|
||||
# "--service-cidr=172.17.0.0/16,fdfd:cafe:00:8001::/112"
|
||||
];
|
||||
nodeLabels = [
|
||||
"node-purpose=kubevirt"
|
||||
];
|
||||
# flannel is disabled; Cilium provides the pod network.
|
||||
};
|
||||
in
|
||||
{
|
||||
imports = (mylib.scanPaths ./.) ++ [
|
||||
@@ -51,7 +22,6 @@ in
|
||||
./hardware-configuration.nix
|
||||
./preservation.nix
|
||||
coreModule
|
||||
k3sModule
|
||||
];
|
||||
|
||||
modules.btrbk.enable = true;
|
||||
|
||||
@@ -9,37 +9,10 @@
|
||||
let
|
||||
hostName = "kubevirt-shushou"; # Define your hostname.
|
||||
|
||||
coreModule = mylib.genKubeVirtHostModule {
|
||||
coreModule = mylib.genVmHostModule {
|
||||
inherit pkgs hostName;
|
||||
inherit (myvars) networking;
|
||||
};
|
||||
k3sModule = mylib.genK3sServerModule {
|
||||
inherit pkgs;
|
||||
kubeconfigFile = "/home/${myvars.username}/.kube/config";
|
||||
tokenFile = "/persistent/kubevirt-k3s-token";
|
||||
# use my own domain & kube-vip's virtual IP for the API server
|
||||
# so that the API server can always be accessed even if some nodes are down
|
||||
masterHost = "kubevirt-cluster-1.writefor.fun";
|
||||
kubeletExtraArgs = [
|
||||
"--cpu-manager-policy=static"
|
||||
# https://kubernetes.io/docs/tasks/administer-cluster/reserve-compute-resources/
|
||||
# we have to reserve some resources for for system daemons running as pods or system services
|
||||
# when cpu-manager's static policy is enabled
|
||||
# the memory we reserved here is also for the kernel, since kernel's memory is not accounted in pods
|
||||
"--system-reserved=cpu=1,memory=2Gi,ephemeral-storage=2Gi"
|
||||
];
|
||||
k3sExtraArgs = [
|
||||
# IPv4 Private CIDR(full) - 172.16.0.0/12
|
||||
# IPv4 Pod CIDR(full) - fdfd:cafe:00:0000::/64 ~ fdfd:cafe:00:7fff::/64
|
||||
# IPv4 Service CIDR(full) - fdfd:cafe:00:8000::/64 ~ fdfd:cafe:00:ffff::/64
|
||||
# "--cluster-cidr=172.16.0.0/16,fdfd:cafe:00:0001::/64"
|
||||
# "--service-cidr=172.17.0.0/16,fdfd:cafe:00:8001::/112"
|
||||
];
|
||||
nodeLabels = [
|
||||
"node-purpose=kubevirt"
|
||||
];
|
||||
# flannel is disabled; Cilium provides the pod network.
|
||||
};
|
||||
in
|
||||
{
|
||||
imports = (mylib.scanPaths ./.) ++ [
|
||||
@@ -48,7 +21,6 @@ in
|
||||
../kubevirt-shoryu/hardware-configuration.nix
|
||||
../kubevirt-shoryu/preservation.nix
|
||||
coreModule
|
||||
k3sModule
|
||||
];
|
||||
|
||||
modules.btrbk.enable = true;
|
||||
|
||||
@@ -9,37 +9,10 @@
|
||||
let
|
||||
hostName = "kubevirt-youko"; # Define your hostname.
|
||||
|
||||
coreModule = mylib.genKubeVirtHostModule {
|
||||
coreModule = mylib.genVmHostModule {
|
||||
inherit pkgs hostName;
|
||||
inherit (myvars) networking;
|
||||
};
|
||||
k3sModule = mylib.genK3sServerModule {
|
||||
inherit pkgs;
|
||||
kubeconfigFile = "/home/${myvars.username}/.kube/config";
|
||||
tokenFile = "/persistent/kubevirt-k3s-token";
|
||||
# use my own domain & kube-vip's virtual IP for the API server
|
||||
# so that the API server can always be accessed even if some nodes are down
|
||||
masterHost = "kubevirt-cluster-1.writefor.fun";
|
||||
kubeletExtraArgs = [
|
||||
"--cpu-manager-policy=static"
|
||||
# https://kubernetes.io/docs/tasks/administer-cluster/reserve-compute-resources/
|
||||
# we have to reserve some resources for for system daemons running as pods or system services
|
||||
# when cpu-manager's static policy is enabled
|
||||
# the memory we reserved here is also for the kernel, since kernel's memory is not accounted in pods
|
||||
"--system-reserved=cpu=1,memory=2Gi,ephemeral-storage=2Gi"
|
||||
];
|
||||
k3sExtraArgs = [
|
||||
# IPv4 Private CIDR(full) - 172.16.0.0/12
|
||||
# IPv4 Pod CIDR(full) - fdfd:cafe:00:0000::/64 ~ fdfd:cafe:00:7fff::/64
|
||||
# IPv4 Service CIDR(full) - fdfd:cafe:00:8000::/64 ~ fdfd:cafe:00:ffff::/64
|
||||
# "--cluster-cidr=172.16.0.0/16,fdfd:cafe:00:0001::/64"
|
||||
# "--service-cidr=172.17.0.0/16,fdfd:cafe:00:8001::/112"
|
||||
];
|
||||
nodeLabels = [
|
||||
"node-purpose=kubevirt"
|
||||
];
|
||||
# flannel is disabled; Cilium provides the pod network.
|
||||
};
|
||||
in
|
||||
{
|
||||
imports = (mylib.scanPaths ./.) ++ [
|
||||
@@ -48,7 +21,6 @@ in
|
||||
../kubevirt-shoryu/hardware-configuration.nix
|
||||
../kubevirt-shoryu/preservation.nix
|
||||
coreModule
|
||||
k3sModule
|
||||
];
|
||||
|
||||
modules.btrbk.enable = true;
|
||||
|
||||
+1
-1
@@ -8,7 +8,7 @@
|
||||
|
||||
genK3sServerModule = import ./genK3sServerModule.nix;
|
||||
genK3sAgentModule = import ./genK3sAgentModule.nix;
|
||||
genKubeVirtHostModule = import ./genKubeVirtHostModule.nix;
|
||||
genVmHostModule = import ./genVmHostModule.nix;
|
||||
genKubeVirtGuestModule = import ./genKubeVirtGuestModule.nix;
|
||||
genMicrovmGuestModule = import ./genMicrovmGuestModule.nix;
|
||||
|
||||
|
||||
@@ -5,7 +5,7 @@
|
||||
...
|
||||
}:
|
||||
let
|
||||
inherit (networking) proxyGateway proxyGateway6 nameservers;
|
||||
inherit (networking) proxyGateway proxyGateway6;
|
||||
inherit (networking.hostsAddr.${hostName}) iface ipv4;
|
||||
ipv4WithMask = "${ipv4}/24";
|
||||
in
|
||||
@@ -20,13 +20,11 @@ in
|
||||
"fat"
|
||||
"vfat"
|
||||
"exfat"
|
||||
"nfs" # required by longhorn
|
||||
"nfs"
|
||||
];
|
||||
|
||||
boot.kernelModules = [
|
||||
"kvm-amd"
|
||||
"vfio-pci"
|
||||
];
|
||||
# KVM for the microVMs and the libvirt domains.
|
||||
boot.kernelModules = [ "kvm-amd" ];
|
||||
boot.extraModprobeConfig = "options kvm_amd nested=1"; # for amd cpu
|
||||
|
||||
boot.kernel.sysctl = {
|
||||
@@ -52,33 +50,18 @@ in
|
||||
|
||||
# NOTE: vm.swappiness is intentionally NOT set here; it comes from
|
||||
# modules/nixos/base/zram.nix (mkDefault 180, tuned for the zram device).
|
||||
# The previous hard-coded 0 disabled swapping entirely, so the zram device
|
||||
# (already enabled by that module) was never used.
|
||||
};
|
||||
|
||||
# zram itself is provided by modules/nixos/base/zram.nix (enabled by default).
|
||||
# Kill the greediest process before the host starts thrashing / gets OOM-killed.
|
||||
# (defaults: trigger at 10% free mem / swap, kill hard at half of that.)
|
||||
services.earlyoom.enable = true;
|
||||
|
||||
environment.systemPackages = with pkgs; [
|
||||
# Validate Hardware Virtualization Support via:
|
||||
# virt-host-validate qemu
|
||||
libvirt
|
||||
kubevirt # virtctl
|
||||
];
|
||||
|
||||
# Workaround for longhorn running on NixOS
|
||||
# https://github.com/longhorn/longhorn/issues/2166
|
||||
systemd.tmpfiles.rules = [
|
||||
"L+ /usr/local/bin - - - - /run/current-system/sw/bin/"
|
||||
];
|
||||
# Longhorn uses open-iscsi to create block devices.
|
||||
services.openiscsi = {
|
||||
name = "iqn.2020-08.org.linux-iscsi.initiatorhost:${hostName}";
|
||||
enable = true;
|
||||
};
|
||||
|
||||
networking = {
|
||||
inherit hostName;
|
||||
|
||||
@@ -89,9 +72,9 @@ in
|
||||
networking.useNetworkd = true;
|
||||
systemd.network.enable = true;
|
||||
|
||||
# Linux bridge for the VMs' secondary network. It replaces the previous OVS
|
||||
# bridge: the `bridge` CNI plugin attaches the VM taps to br0, and the host
|
||||
# keeps its own address on br0 (the physical NIC is just a bridge port).
|
||||
# Linux bridge for the VMs: the microVM taps and the libvirt domains attach to
|
||||
# br0, and the host keeps its own address on it (the physical NIC is just a
|
||||
# bridge port).
|
||||
systemd.network.netdevs."10-br0" = {
|
||||
netdevConfig = {
|
||||
Name = "br0";
|
||||
@@ -99,8 +82,6 @@ in
|
||||
};
|
||||
};
|
||||
|
||||
# systemd.services."systemd-networkd".environment.SYSTEMD_LOG_LEVEL = "debug";
|
||||
|
||||
# Set the host's address on the bridge interface instead of the physical interface!
|
||||
systemd.network.networks = {
|
||||
"10-br0" = {
|
||||
Reference in new issue
Block a user