From 0004bccc9dbe1d34049f933bd8513df503782fb0 Mon Sep 17 00:00:00 2001 From: Ryan Yin Date: Sun, 29 Jun 2025 15:00:53 +0800 Subject: [PATCH] feat: hosts/k8s - fix k3s cidr (#200) --- hosts/k8s/k3s-prod-1-master-1/default.nix | 15 +++++++-------- hosts/k8s/k3s-prod-1-master-2/default.nix | 14 +++++++------- hosts/k8s/k3s-prod-1-master-3/default.nix | 14 +++++++------- hosts/k8s/k3s-prod-1-worker-1/default.nix | 14 +++++++------- hosts/k8s/k3s-prod-1-worker-2/default.nix | 14 +++++++------- hosts/k8s/k3s-prod-1-worker-3/default.nix | 14 +++++++------- hosts/k8s/k3s-test-1-master-1/default.nix | 2 +- hosts/k8s/k3s-test-1-master-2/default.nix | 2 +- hosts/k8s/k3s-test-1-master-3/default.nix | 2 +- hosts/k8s/kubevirt-shoryu/default.nix | 7 ++++--- hosts/k8s/kubevirt-shushou/default.nix | 7 ++++--- hosts/k8s/kubevirt-youko/default.nix | 7 ++++--- lib/genK3sAgentModule.nix | 11 +++++++++-- lib/genK3sServerModule.nix | 11 +++++++++-- 14 files changed, 75 insertions(+), 59 deletions(-) diff --git a/hosts/k8s/k3s-prod-1-master-1/default.nix b/hosts/k8s/k3s-prod-1-master-1/default.nix index e7958cd6..125ada7a 100644 --- a/hosts/k8s/k3s-prod-1-master-1/default.nix +++ b/hosts/k8s/k3s-prod-1-master-1/default.nix @@ -20,14 +20,13 @@ # use my own domain & kube-vip's virtual IP for the API server # so that the API server can always be accessed even if some nodes are down masterHost = "prod-cluster-1.writefor.fun"; - - kubeletExtraArgs = [ - # IPv4 Private CIDR(full) - 172.16.0.0/12 - # IPv4 Pod CIDR(full) - fdfd:cafe:00:0000::/64 ~ fdfd:cafe:00:7fff::/64 - # IPv4 Service CIDR(full) - fdfd:cafe:00:8000::/64 ~ fdfd:cafe:00:ffff::/64 - "--cluster-cidr=172.20.0.0/16,fdfd:cafe:00:0003::/64" - "--service-cidr=172.21.0.0/16,fdfd:cafe:00:8003::/112" - ]; + # k3sExtraArgs = [ + # # IPv4 Private CIDR(full) - 172.16.0.0/12 + # # IPv4 Pod CIDR(full) - fdfd:cafe:00:0000::/64 ~ fdfd:cafe:00:7fff::/64 + # # IPv4 Service CIDR(full) - fdfd:cafe:00:8000::/64 ~ fdfd:cafe:00:ffff::/64 + # "--cluster-cidr=172.20.0.0/16,fdfd:cafe:00:0003::/64" + # "--service-cidr=172.21.0.0/16,fdfd:cafe:00:8003::/112" + # ]; }; in { imports = diff --git a/hosts/k8s/k3s-prod-1-master-2/default.nix b/hosts/k8s/k3s-prod-1-master-2/default.nix index 772d2f99..e8b0f027 100644 --- a/hosts/k8s/k3s-prod-1-master-2/default.nix +++ b/hosts/k8s/k3s-prod-1-master-2/default.nix @@ -19,13 +19,13 @@ # so that the API server can always be accessed even if some nodes are down masterHost = "prod-cluster-1.writefor.fun"; - kubeletExtraArgs = [ - # IPv4 Private CIDR(full) - 172.16.0.0/12 - # IPv4 Pod CIDR(full) - fdfd:cafe:00:0000::/64 ~ fdfd:cafe:00:7fff::/64 - # IPv4 Service CIDR(full) - fdfd:cafe:00:8000::/64 ~ fdfd:cafe:00:ffff::/64 - "--cluster-cidr=172.20.0.0/16,fdfd:cafe:00:0003::/64" - "--service-cidr=172.21.0.0/16,fdfd:cafe:00:8003::/112" - ]; + # k3sExtraArgs = [ + # # IPv4 Private CIDR(full) - 172.16.0.0/12 + # # IPv4 Pod CIDR(full) - fdfd:cafe:00:0000::/64 ~ fdfd:cafe:00:7fff::/64 + # # IPv4 Service CIDR(full) - fdfd:cafe:00:8000::/64 ~ fdfd:cafe:00:ffff::/64 + # "--cluster-cidr=172.20.0.0/16,fdfd:cafe:00:0003::/64" + # "--service-cidr=172.21.0.0/16,fdfd:cafe:00:8003::/112" + # ]; }; in { imports = diff --git a/hosts/k8s/k3s-prod-1-master-3/default.nix b/hosts/k8s/k3s-prod-1-master-3/default.nix index 57184504..6ebb0ef6 100644 --- a/hosts/k8s/k3s-prod-1-master-3/default.nix +++ b/hosts/k8s/k3s-prod-1-master-3/default.nix @@ -19,13 +19,13 @@ # so that the API server can always be accessed even if some nodes are down masterHost = "prod-cluster-1.writefor.fun"; - kubeletExtraArgs = [ - # IPv4 Private CIDR(full) - 172.16.0.0/12 - # IPv4 Pod CIDR(full) - fdfd:cafe:00:0000::/64 ~ fdfd:cafe:00:7fff::/64 - # IPv4 Service CIDR(full) - fdfd:cafe:00:8000::/64 ~ fdfd:cafe:00:ffff::/64 - "--cluster-cidr=172.20.0.0/16,fdfd:cafe:00:0003::/64" - "--service-cidr=172.21.0.0/16,fdfd:cafe:00:8003::/112" - ]; + # k3sExtraArgs = [ + # # IPv4 Private CIDR(full) - 172.16.0.0/12 + # # IPv4 Pod CIDR(full) - fdfd:cafe:00:0000::/64 ~ fdfd:cafe:00:7fff::/64 + # # IPv4 Service CIDR(full) - fdfd:cafe:00:8000::/64 ~ fdfd:cafe:00:ffff::/64 + # "--cluster-cidr=172.20.0.0/16,fdfd:cafe:00:0003::/64" + # "--service-cidr=172.21.0.0/16,fdfd:cafe:00:8003::/112" + # ]; }; in { imports = diff --git a/hosts/k8s/k3s-prod-1-worker-1/default.nix b/hosts/k8s/k3s-prod-1-worker-1/default.nix index c87ca4a6..f1a3fdd5 100644 --- a/hosts/k8s/k3s-prod-1-worker-1/default.nix +++ b/hosts/k8s/k3s-prod-1-worker-1/default.nix @@ -18,13 +18,13 @@ # so that the API server can always be accessed even if some nodes are down masterHost = "prod-cluster-1.writefor.fun"; - kubeletExtraArgs = [ - # IPv4 Private CIDR(full) - 172.16.0.0/12 - # IPv4 Pod CIDR(full) - fdfd:cafe:00:0000::/64 ~ fdfd:cafe:00:7fff::/64 - # IPv4 Service CIDR(full) - fdfd:cafe:00:8000::/64 ~ fdfd:cafe:00:ffff::/64 - "--cluster-cidr=172.20.0.0/16,fdfd:cafe:00:0003::/64" - "--service-cidr=172.21.0.0/16,fdfd:cafe:00:8003::/112" - ]; + # k3sExtraArgs = [ + # # IPv4 Private CIDR(full) - 172.16.0.0/12 + # # IPv4 Pod CIDR(full) - fdfd:cafe:00:0000::/64 ~ fdfd:cafe:00:7fff::/64 + # # IPv4 Service CIDR(full) - fdfd:cafe:00:8000::/64 ~ fdfd:cafe:00:ffff::/64 + # "--cluster-cidr=172.20.0.0/16,fdfd:cafe:00:0003::/64" + # "--service-cidr=172.21.0.0/16,fdfd:cafe:00:8003::/112" + # ]; }; in { imports = diff --git a/hosts/k8s/k3s-prod-1-worker-2/default.nix b/hosts/k8s/k3s-prod-1-worker-2/default.nix index ff9ed740..b8f4b9c8 100644 --- a/hosts/k8s/k3s-prod-1-worker-2/default.nix +++ b/hosts/k8s/k3s-prod-1-worker-2/default.nix @@ -18,13 +18,13 @@ # so that the API server can always be accessed even if some nodes are down masterHost = "prod-cluster-1.writefor.fun"; - kubeletExtraArgs = [ - # IPv4 Private CIDR(full) - 172.16.0.0/12 - # IPv4 Pod CIDR(full) - fdfd:cafe:00:0000::/64 ~ fdfd:cafe:00:7fff::/64 - # IPv4 Service CIDR(full) - fdfd:cafe:00:8000::/64 ~ fdfd:cafe:00:ffff::/64 - "--cluster-cidr=172.20.0.0/16,fdfd:cafe:00:0003::/64" - "--service-cidr=172.21.0.0/16,fdfd:cafe:00:8003::/112" - ]; + # k3sExtraArgs = [ + # # IPv4 Private CIDR(full) - 172.16.0.0/12 + # # IPv4 Pod CIDR(full) - fdfd:cafe:00:0000::/64 ~ fdfd:cafe:00:7fff::/64 + # # IPv4 Service CIDR(full) - fdfd:cafe:00:8000::/64 ~ fdfd:cafe:00:ffff::/64 + # "--cluster-cidr=172.20.0.0/16,fdfd:cafe:00:0003::/64" + # "--service-cidr=172.21.0.0/16,fdfd:cafe:00:8003::/112" + # ]; }; in { imports = diff --git a/hosts/k8s/k3s-prod-1-worker-3/default.nix b/hosts/k8s/k3s-prod-1-worker-3/default.nix index 25ed9dc4..899f7f5b 100644 --- a/hosts/k8s/k3s-prod-1-worker-3/default.nix +++ b/hosts/k8s/k3s-prod-1-worker-3/default.nix @@ -18,13 +18,13 @@ # so that the API server can always be accessed even if some nodes are down masterHost = "prod-cluster-1.writefor.fun"; - kubeletExtraArgs = [ - # IPv4 Private CIDR(full) - 172.16.0.0/12 - # IPv4 Pod CIDR(full) - fdfd:cafe:00:0000::/64 ~ fdfd:cafe:00:7fff::/64 - # IPv4 Service CIDR(full) - fdfd:cafe:00:8000::/64 ~ fdfd:cafe:00:ffff::/64 - "--cluster-cidr=172.20.0.0/16,fdfd:cafe:00:0003::/64" - "--service-cidr=172.21.0.0/16,fdfd:cafe:00:8003::/112" - ]; + # k3sExtraArgs = [ + # # IPv4 Private CIDR(full) - 172.16.0.0/12 + # # IPv4 Pod CIDR(full) - fdfd:cafe:00:0000::/64 ~ fdfd:cafe:00:7fff::/64 + # # IPv4 Service CIDR(full) - fdfd:cafe:00:8000::/64 ~ fdfd:cafe:00:ffff::/64 + # "--cluster-cidr=172.20.0.0/16,fdfd:cafe:00:0003::/64" + # "--service-cidr=172.21.0.0/16,fdfd:cafe:00:8003::/112" + # ]; }; in { imports = diff --git a/hosts/k8s/k3s-test-1-master-1/default.nix b/hosts/k8s/k3s-test-1-master-1/default.nix index 80332a87..d0800036 100644 --- a/hosts/k8s/k3s-test-1-master-1/default.nix +++ b/hosts/k8s/k3s-test-1-master-1/default.nix @@ -21,7 +21,7 @@ # so that the API server can always be accessed even if some nodes are down masterHost = "test-cluster-1.writefor.fun"; - # kubeletExtraArgs = [ + # k3sExtraArgs = [ # # IPv4 Private CIDR(full) - 172.16.0.0/12 # # IPv4 Pod CIDR(full) - fdfd:cafe:00:0000::/64 ~ fdfd:cafe:00:7fff::/64 # # IPv4 Service CIDR(full) - fdfd:cafe:00:8000::/64 ~ fdfd:cafe:00:ffff::/64 diff --git a/hosts/k8s/k3s-test-1-master-2/default.nix b/hosts/k8s/k3s-test-1-master-2/default.nix index c149ebb2..3be0ab3b 100644 --- a/hosts/k8s/k3s-test-1-master-2/default.nix +++ b/hosts/k8s/k3s-test-1-master-2/default.nix @@ -19,7 +19,7 @@ # so that the API server can always be accessed even if some nodes are down masterHost = "test-cluster-1.writefor.fun"; - # kubeletExtraArgs = [ + # k3sExtraArgs = [ # # IPv4 Private CIDR(full) - 172.16.0.0/12 # # IPv4 Pod CIDR(full) - fdfd:cafe:00:0000::/64 ~ fdfd:cafe:00:7fff::/64 # # IPv4 Service CIDR(full) - fdfd:cafe:00:8000::/64 ~ fdfd:cafe:00:ffff::/64 diff --git a/hosts/k8s/k3s-test-1-master-3/default.nix b/hosts/k8s/k3s-test-1-master-3/default.nix index dfcf2b64..c3d4cdc8 100644 --- a/hosts/k8s/k3s-test-1-master-3/default.nix +++ b/hosts/k8s/k3s-test-1-master-3/default.nix @@ -19,7 +19,7 @@ # so that the API server can always be accessed even if some nodes are down masterHost = "test-cluster-1.writefor.fun"; - # kubeletExtraArgs = [ + # k3sExtraArgs = [ # # IPv4 Private CIDR(full) - 172.16.0.0/12 # # IPv4 Pod CIDR(full) - fdfd:cafe:00:0000::/64 ~ fdfd:cafe:00:7fff::/64 # # IPv4 Service CIDR(full) - fdfd:cafe:00:8000::/64 ~ fdfd:cafe:00:ffff::/64 diff --git a/hosts/k8s/kubevirt-shoryu/default.nix b/hosts/k8s/kubevirt-shoryu/default.nix index b580c331..a3e13d6e 100644 --- a/hosts/k8s/kubevirt-shoryu/default.nix +++ b/hosts/k8s/kubevirt-shoryu/default.nix @@ -29,12 +29,13 @@ # when cpu-manager's static policy is enabled # the memory we reserved here is also for the kernel, since kernel's memory is not accounted in pods "--system-reserved=cpu=1,memory=2Gi,ephemeral-storage=2Gi" - + ]; + k3sExtraArgs = [ # IPv4 Private CIDR(full) - 172.16.0.0/12 # IPv4 Pod CIDR(full) - fdfd:cafe:00:0000::/64 ~ fdfd:cafe:00:7fff::/64 # IPv4 Service CIDR(full) - fdfd:cafe:00:8000::/64 ~ fdfd:cafe:00:ffff::/64 - "--cluster-cidr=172.16.0.0/16,fdfd:cafe:00:0001::/64" - "--service-cidr=172.17.0.0/16,fdfd:cafe:00:8001::/112" + # "--cluster-cidr=172.16.0.0/16,fdfd:cafe:00:0001::/64" + # "--service-cidr=172.17.0.0/16,fdfd:cafe:00:8001::/112" ]; nodeLabels = [ "node-purpose=kubevirt" diff --git a/hosts/k8s/kubevirt-shushou/default.nix b/hosts/k8s/kubevirt-shushou/default.nix index 84da6742..784d304a 100644 --- a/hosts/k8s/kubevirt-shushou/default.nix +++ b/hosts/k8s/kubevirt-shushou/default.nix @@ -26,12 +26,13 @@ # when cpu-manager's static policy is enabled # the memory we reserved here is also for the kernel, since kernel's memory is not accounted in pods "--system-reserved=cpu=1,memory=2Gi,ephemeral-storage=2Gi" - + ]; + k3sExtraArgs = [ # IPv4 Private CIDR(full) - 172.16.0.0/12 # IPv4 Pod CIDR(full) - fdfd:cafe:00:0000::/64 ~ fdfd:cafe:00:7fff::/64 # IPv4 Service CIDR(full) - fdfd:cafe:00:8000::/64 ~ fdfd:cafe:00:ffff::/64 - "--cluster-cidr=172.16.0.0/16,fdfd:cafe:00:0001::/64" - "--service-cidr=172.17.0.0/16,fdfd:cafe:00:8001::/112" + # "--cluster-cidr=172.16.0.0/16,fdfd:cafe:00:0001::/64" + # "--service-cidr=172.17.0.0/16,fdfd:cafe:00:8001::/112" ]; nodeLabels = [ "node-purpose=kubevirt" diff --git a/hosts/k8s/kubevirt-youko/default.nix b/hosts/k8s/kubevirt-youko/default.nix index 501081f9..4262e5e6 100644 --- a/hosts/k8s/kubevirt-youko/default.nix +++ b/hosts/k8s/kubevirt-youko/default.nix @@ -26,12 +26,13 @@ # when cpu-manager's static policy is enabled # the memory we reserved here is also for the kernel, since kernel's memory is not accounted in pods "--system-reserved=cpu=1,memory=2Gi,ephemeral-storage=2Gi" - + ]; + k3sExtraArgs = [ # IPv4 Private CIDR(full) - 172.16.0.0/12 # IPv4 Pod CIDR(full) - fdfd:cafe:00:0000::/64 ~ fdfd:cafe:00:7fff::/64 # IPv4 Service CIDR(full) - fdfd:cafe:00:8000::/64 ~ fdfd:cafe:00:ffff::/64 - "--cluster-cidr=172.16.0.0/16,fdfd:cafe:00:0001::/64" - "--service-cidr=172.17.0.0/16,fdfd:cafe:00:8001::/112" + # "--cluster-cidr=172.16.0.0/16,fdfd:cafe:00:0001::/64" + # "--service-cidr=172.17.0.0/16,fdfd:cafe:00:8001::/112" ]; nodeLabels = [ "node-purpose=kubevirt" diff --git a/lib/genK3sAgentModule.nix b/lib/genK3sAgentModule.nix index ee4124b0..336295db 100644 --- a/lib/genK3sAgentModule.nix +++ b/lib/genK3sAgentModule.nix @@ -3,6 +3,7 @@ masterHost, tokenFile, nodeLabels ? [], + k3sExtraArgs ? [], ... }: let package = pkgs.k3s; @@ -10,7 +11,12 @@ in { environment.systemPackages = [package]; # Kernel modules required by cilium - boot.kernelModules = ["ip6_tables" "ip6table_mangle" "ip6table_raw" "ip6table_filter"]; + boot.kernelModules = [ + "ip6_tables" + "ip6table_mangle" + "ip6table_raw" + "ip6table_filter" + ]; networking.enableIPv6 = true; networking.nat = { enable = true; @@ -29,7 +35,8 @@ in { [ "--data-dir /var/lib/rancher/k3s" ] - ++ (map (label: "--node-label=${label}") nodeLabels); + ++ (map (label: "--node-label=${label}") nodeLabels) + ++ k3sExtraArgs; in pkgs.lib.concatStringsSep " " flagList; }; diff --git a/lib/genK3sServerModule.nix b/lib/genK3sServerModule.nix index 6b50b2dc..e388eafe 100644 --- a/lib/genK3sServerModule.nix +++ b/lib/genK3sServerModule.nix @@ -11,6 +11,7 @@ masterHost, clusterInit ? false, kubeletExtraArgs ? [], + k3sExtraArgs ? [], nodeLabels ? [], nodeTaints ? [], disableFlannel ? true, @@ -35,7 +36,12 @@ in { ]; # Kernel modules required by cilium - boot.kernelModules = ["ip6_tables" "ip6table_mangle" "ip6table_raw" "ip6table_filter"]; + boot.kernelModules = [ + "ip6_tables" + "ip6table_mangle" + "ip6table_raw" + "ip6table_filter" + ]; networking.enableIPv6 = true; networking.nat = { enable = true; @@ -71,7 +77,8 @@ in { ++ (map (label: "--node-label=${label}") nodeLabels) ++ (map (taint: "--node-taint=${taint}") nodeTaints) ++ (map (arg: "--kubelet-arg=${arg}") kubeletExtraArgs) - ++ (lib.optionals disableFlannel ["--flannel-backend=none"]); + ++ (lib.optionals disableFlannel ["--flannel-backend=none"]) + ++ k3sExtraArgs; in lib.concatStringsSep " " flagList; };