starred/nix-config
1
0
Fork 0
mirror of https://github.com/ryan4yin/nix-config.git synced 2026-07-12 07:32:40 +02:00
Code Issues Packages Projects Releases 10 Wiki Activity

feat(hosts,lib): use kube-vip for k3s's HA

Browse Source
This commit is contained in:
Ryan Yin
2024-03-29 22:34:09 +08:00
parent 27959277d3
commit 6ae98566ed
14 changed files with 45 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files
hosts/k8s/k3s-prod-1-master-1/default.nix
+3
View File
@@ -17,6 +17,9 @@
tokenFile = config.age.secrets."k3s-prod-1-token".path; tokenFile = config.age.secrets."k3s-prod-1-token".path;
# the first node in the cluster should be the one to initialize the cluster # the first node in the cluster should be the one to initialize the cluster
clusterInit = true; clusterInit = true;
# use my own domain & kube-vip's virtual IP for the API server
# so that the API server can always be accessed even if some nodes are down
masterHost = "prod-cluster-1.writefor.fun";
}; };
in { in {
imports = imports =
hosts/k8s/k3s-prod-1-master-2/default.nix
+3 -2
View File
@@ -6,7 +6,6 @@
... ...
}: let }: let
hostName = "k3s-prod-1-master-2"; # define your hostname. hostName = "k3s-prod-1-master-2"; # define your hostname.
k3sServerName = "k3s-prod-1-master-1";
coreModule = mylib.genKubeVirtCoreModule { coreModule = mylib.genKubeVirtCoreModule {
inherit pkgs hostName; inherit pkgs hostName;
@@ -16,7 +15,9 @@
inherit pkgs; inherit pkgs;
kubeconfigFile = "/home/${myvars.username}/.kube/config"; kubeconfigFile = "/home/${myvars.username}/.kube/config";
tokenFile = config.age.secrets."k3s-prod-1-token".path; tokenFile = config.age.secrets."k3s-prod-1-token".path;
serverIp = myvars.networking.hostsAddr.${k3sServerName}.ipv4; # use my own domain & kube-vip's virtual IP for the API server
# so that the API server can always be accessed even if some nodes are down
masterHost = "prod-cluster-1.writefor.fun";
}; };
in { in {
imports = imports =
hosts/k8s/k3s-prod-1-master-3/default.nix
+3 -2
View File
@@ -6,7 +6,6 @@
... ...
}: let }: let
hostName = "k3s-prod-1-master-3"; # define your hostname. hostName = "k3s-prod-1-master-3"; # define your hostname.
k3sServerName = "k3s-prod-1-master-1";
coreModule = mylib.genKubeVirtCoreModule { coreModule = mylib.genKubeVirtCoreModule {
inherit pkgs hostName; inherit pkgs hostName;
@@ -16,7 +15,9 @@
inherit pkgs; inherit pkgs;
kubeconfigFile = "/home/${myvars.username}/.kube/config"; kubeconfigFile = "/home/${myvars.username}/.kube/config";
tokenFile = config.age.secrets."k3s-prod-1-token".path; tokenFile = config.age.secrets."k3s-prod-1-token".path;
serverIp = myvars.networking.hostsAddr.${k3sServerName}.ipv4; # use my own domain & kube-vip's virtual IP for the API server
# so that the API server can always be accessed even if some nodes are down
masterHost = "prod-cluster-1.writefor.fun";
}; };
in { in {
imports = imports =
hosts/k8s/k3s-prod-1-worker-1/default.nix
+3 -2
View File
@@ -6,7 +6,6 @@
... ...
}: let }: let
hostName = "k3s-prod-1-worker-1"; # define your hostname. hostName = "k3s-prod-1-worker-1"; # define your hostname.
k3sServerName = "k3s-prod-1-master-1";
coreModule = mylib.genKubeVirtCoreModule { coreModule = mylib.genKubeVirtCoreModule {
inherit pkgs hostName; inherit pkgs hostName;
@@ -15,7 +14,9 @@
k3sModule = mylib.genK3sAgentModule { k3sModule = mylib.genK3sAgentModule {
inherit pkgs; inherit pkgs;
tokenFile = config.age.secrets."k3s-prod-1-token".path; tokenFile = config.age.secrets."k3s-prod-1-token".path;
serverIp = myvars.networking.hostsAddr.${k3sServerName}.ipv4; # use my own domain & kube-vip's virtual IP for the API server
# so that the API server can always be accessed even if some nodes are down
masterHost = "prod-cluster-1.writefor.fun";
}; };
in { in {
imports = imports =
hosts/k8s/k3s-prod-1-worker-2/default.nix
+3 -2
View File
@@ -6,7 +6,6 @@
... ...
}: let }: let
hostName = "k3s-prod-1-worker-2"; # define your hostname. hostName = "k3s-prod-1-worker-2"; # define your hostname.
k3sServerName = "k3s-prod-1-master-1";
coreModule = mylib.genKubeVirtCoreModule { coreModule = mylib.genKubeVirtCoreModule {
inherit pkgs hostName; inherit pkgs hostName;
@@ -15,7 +14,9 @@
k3sModule = mylib.genK3sAgentModule { k3sModule = mylib.genK3sAgentModule {
inherit pkgs; inherit pkgs;
tokenFile = config.age.secrets."k3s-prod-1-token".path; tokenFile = config.age.secrets."k3s-prod-1-token".path;
serverIp = myvars.networking.hostsAddr.${k3sServerName}.ipv4; # use my own domain & kube-vip's virtual IP for the API server
# so that the API server can always be accessed even if some nodes are down
masterHost = "prod-cluster-1.writefor.fun";
}; };
in { in {
imports = imports =
hosts/k8s/k3s-prod-1-worker-3/default.nix
+3 -2
View File
@@ -6,7 +6,6 @@
... ...
}: let }: let
hostName = "k3s-prod-1-worker-3"; # define your hostname. hostName = "k3s-prod-1-worker-3"; # define your hostname.
k3sServerName = "k3s-prod-1-master-1";
coreModule = mylib.genKubeVirtCoreModule { coreModule = mylib.genKubeVirtCoreModule {
inherit pkgs hostName; inherit pkgs hostName;
@@ -15,7 +14,9 @@
k3sModule = mylib.genK3sAgentModule { k3sModule = mylib.genK3sAgentModule {
inherit pkgs; inherit pkgs;
tokenFile = config.age.secrets."k3s-prod-1-token".path; tokenFile = config.age.secrets."k3s-prod-1-token".path;
serverIp = myvars.networking.hostsAddr.${k3sServerName}.ipv4; # use my own domain & kube-vip's virtual IP for the API server
# so that the API server can always be accessed even if some nodes are down
masterHost = "prod-cluster-1.writefor.fun";
}; };
in { in {
imports = imports =
hosts/k8s/k3s-test-1-master-1/default.nix
+3
View File
@@ -17,6 +17,9 @@
tokenFile = config.age.secrets."k3s-prod-1-token".path; tokenFile = config.age.secrets."k3s-prod-1-token".path;
# the first node in the cluster should be the one to initialize the cluster # the first node in the cluster should be the one to initialize the cluster
clusterInit = true; clusterInit = true;
# use my own domain & kube-vip's virtual IP for the API server
# so that the API server can always be accessed even if some nodes are down
masterHost = "test-cluster-1.writefor.fun";
}; };
in { in {
imports = imports =
hosts/k8s/k3s-test-1-master-2/default.nix
+3 -2
View File
@@ -6,7 +6,6 @@
... ...
}: let }: let
hostName = "k3s-test-1-master-2"; # define your hostname. hostName = "k3s-test-1-master-2"; # define your hostname.
k3sServerName = "k3s-test-1-master-1";
coreModule = mylib.genKubeVirtCoreModule { coreModule = mylib.genKubeVirtCoreModule {
inherit pkgs hostName; inherit pkgs hostName;
@@ -16,7 +15,9 @@
inherit pkgs; inherit pkgs;
kubeconfigFile = "/home/${myvars.username}/.kube/config"; kubeconfigFile = "/home/${myvars.username}/.kube/config";
tokenFile = config.age.secrets."k3s-prod-1-token".path; tokenFile = config.age.secrets."k3s-prod-1-token".path;
serverIp = myvars.networking.hostsAddr.${k3sServerName}.ipv4; # use my own domain & kube-vip's virtual IP for the API server
# so that the API server can always be accessed even if some nodes are down
masterHost = "test-cluster-1.writefor.fun";
}; };
in { in {
imports = imports =
hosts/k8s/k3s-test-1-master-3/default.nix
+3 -2
View File
@@ -6,7 +6,6 @@
... ...
}: let }: let
hostName = "k3s-test-1-master-3"; # define your hostname. hostName = "k3s-test-1-master-3"; # define your hostname.
k3sServerName = "k3s-test-1-master-1";
coreModule = mylib.genKubeVirtCoreModule { coreModule = mylib.genKubeVirtCoreModule {
inherit pkgs hostName; inherit pkgs hostName;
@@ -16,7 +15,9 @@
inherit pkgs; inherit pkgs;
kubeconfigFile = "/home/${myvars.username}/.kube/config"; kubeconfigFile = "/home/${myvars.username}/.kube/config";
tokenFile = config.age.secrets."k3s-prod-1-token".path; tokenFile = config.age.secrets."k3s-prod-1-token".path;
serverIp = myvars.networking.hostsAddr.${k3sServerName}.ipv4; # use my own domain & kube-vip's virtual IP for the API server
# so that the API server can always be accessed even if some nodes are down
masterHost = "test-cluster-1.writefor.fun";
}; };
in { in {
imports = imports =
hosts/k8s/kubevirt-shoryu/default.nix
+3
View File
@@ -19,6 +19,9 @@
tokenFile = "/run/media/nixos_k3s/kubevirt-k3s-token"; tokenFile = "/run/media/nixos_k3s/kubevirt-k3s-token";
# the first node in the cluster should be the one to initialize the cluster # the first node in the cluster should be the one to initialize the cluster
clusterInit = true; clusterInit = true;
# use my own domain & kube-vip's virtual IP for the API server
# so that the API server can always be accessed even if some nodes are down
masterHost = "kubevirt-cluster-1.writefor.fun";
}; };
in { in {
imports = imports =
hosts/k8s/kubevirt-shushou/default.nix
+3 -1
View File
@@ -17,7 +17,9 @@
inherit pkgs; inherit pkgs;
kubeconfigFile = "/home/${myvars.username}/.kube/config"; kubeconfigFile = "/home/${myvars.username}/.kube/config";
tokenFile = "/run/media/nixos_k3s/kubevirt-k3s-token"; tokenFile = "/run/media/nixos_k3s/kubevirt-k3s-token";
serverIp = myvars.networking.hostsAddr.${k3sServerName}.ipv4; # use my own domain & kube-vip's virtual IP for the API server
# so that the API server can always be accessed even if some nodes are down
masterHost = "kubevirt-cluster-1.writefor.fun";
}; };
in { in {
imports = imports =
hosts/k8s/kubevirt-youko/default.nix
+3 -1
View File
@@ -17,7 +17,9 @@
inherit pkgs; inherit pkgs;
kubeconfigFile = "/home/${myvars.username}/.kube/config"; kubeconfigFile = "/home/${myvars.username}/.kube/config";
tokenFile = "/run/media/nixos_k3s/kubevirt-k3s-token"; tokenFile = "/run/media/nixos_k3s/kubevirt-k3s-token";
serverIp = myvars.networking.hostsAddr.${k3sServerName}.ipv4; # use my own domain & kube-vip's virtual IP for the API server
# so that the API server can always be accessed even if some nodes are down
masterHost = "kubevirt-cluster-1.writefor.fun";
}; };
in { in {
imports = imports =
lib/genK3sAgentModule.nix
+2 -2
View File
@@ -1,6 +1,6 @@
{ {
pkgs, pkgs,
serverIp, masterHost,
tokenFile, tokenFile,
... ...
}: let }: let
@@ -12,7 +12,7 @@ in {
inherit package tokenFile; inherit package tokenFile;
role = "agent"; role = "agent";
serverAddr = "https://${serverIp}:6443"; serverAddr = "https://${masterHost}:6443";
# https://docs.k3s.io/cli/agent # https://docs.k3s.io/cli/agent
extraFlags = let extraFlags = let
flagList = [ flagList = [
lib/genK3sServerModule.nix
+7 -4
View File
@@ -6,8 +6,10 @@
# If you are configuring an HA cluster with an embedded etcd, # If you are configuring an HA cluster with an embedded etcd,
# the 1st server must have `clusterInit = true` # the 1st server must have `clusterInit = true`
# and other servers must connect to it using `serverAddr`. # and other servers must connect to it using `serverAddr`.
serverIp ? null, #
clusterInit ? (serverIp == null), # this can be a domain name or an IP address(such as kube-vip's virtual IP)
masterHost,
clusterInit ? false,
addTaints ? false, addTaints ? false,
... ...
}: let }: let
@@ -32,14 +34,14 @@ in {
serverAddr = serverAddr =
if clusterInit if clusterInit
then "" then ""
else "https://${serverIp}:6443"; else "https://${masterHost}:6443";
role = "server"; role = "server";
# https://docs.k3s.io/cli/server # https://docs.k3s.io/cli/server
extraFlags = let extraFlags = let
flagList = flagList =
[ [
"--write-kubeconfig ${kubeconfigFile}" "--write-kubeconfig=${kubeconfigFile}"
"--write-kubeconfig-mode=644" "--write-kubeconfig-mode=644"
"--service-node-port-range=80-32767" "--service-node-port-range=80-32767"
"--kube-apiserver-arg='--allow-privileged=true'" # required by kubevirt "--kube-apiserver-arg='--allow-privileged=true'" # required by kubevirt
@@ -52,6 +54,7 @@ in {
"--disable=servicelb" # we use kube-vip instead "--disable=servicelb" # we use kube-vip instead
"--flannel-backend=none" # we use cilium instead "--flannel-backend=none" # we use cilium instead
"--disable-network-policy" "--disable-network-policy"
"--tls-san=${masterHost}"
] ]
# prevent workloads from running on the master # prevent workloads from running on the master
++ (pkgs.lib.optionals addTaints ["--node-taint=CriticalAddonsOnly=true:NoExecute"]); ++ (pkgs.lib.optionals addTaints ["--node-taint=CriticalAddonsOnly=true:NoExecute"]);