starred/nix-config
1
0
Fork 0
mirror of https://github.com/ryan4yin/nix-config.git synced 2026-01-11 20:40:24 +01:00
Code Issues Packages Projects Releases 10 Wiki Activity

feat(hosts,lib): use kube-vip for k3s's HA

Browse Source
This commit is contained in:
Ryan Yin
2024-03-29 22:34:09 +08:00
parent 27959277d3
commit 6ae98566ed
14 changed files with 45 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
hosts/k8s/k3s-prod-1-master-1/default.nix
View File

@@ -17,6 +17,9 @@
tokenFile = config.age.secrets."k3s-prod-1-token".path;
# the first node in the cluster should be the one to initialize the cluster
clusterInit = true;
# use my own domain & kube-vip's virtual IP for the API server
# so that the API server can always be accessed even if some nodes are down
masterHost = "prod-cluster-1.writefor.fun";
};
in {
imports =

5
hosts/k8s/k3s-prod-1-master-2/default.nix
View File

@@ -6,7 +6,6 @@
...
}: let
hostName = "k3s-prod-1-master-2"; # define your hostname.
k3sServerName = "k3s-prod-1-master-1";
coreModule = mylib.genKubeVirtCoreModule {
inherit pkgs hostName;
@@ -16,7 +15,9 @@
inherit pkgs;
kubeconfigFile = "/home/${myvars.username}/.kube/config";
tokenFile = config.age.secrets."k3s-prod-1-token".path;
serverIp = myvars.networking.hostsAddr.${k3sServerName}.ipv4;
# use my own domain & kube-vip's virtual IP for the API server
# so that the API server can always be accessed even if some nodes are down
masterHost = "prod-cluster-1.writefor.fun";
};
in {
imports =

5
hosts/k8s/k3s-prod-1-master-3/default.nix
View File

@@ -6,7 +6,6 @@
...
}: let
hostName = "k3s-prod-1-master-3"; # define your hostname.
k3sServerName = "k3s-prod-1-master-1";
coreModule = mylib.genKubeVirtCoreModule {
inherit pkgs hostName;
@@ -16,7 +15,9 @@
inherit pkgs;
kubeconfigFile = "/home/${myvars.username}/.kube/config";
tokenFile = config.age.secrets."k3s-prod-1-token".path;
serverIp = myvars.networking.hostsAddr.${k3sServerName}.ipv4;
# use my own domain & kube-vip's virtual IP for the API server
# so that the API server can always be accessed even if some nodes are down
masterHost = "prod-cluster-1.writefor.fun";
};
in {
imports =

5
hosts/k8s/k3s-prod-1-worker-1/default.nix
View File

@@ -6,7 +6,6 @@
...
}: let
hostName = "k3s-prod-1-worker-1"; # define your hostname.
k3sServerName = "k3s-prod-1-master-1";
coreModule = mylib.genKubeVirtCoreModule {
inherit pkgs hostName;
@@ -15,7 +14,9 @@
k3sModule = mylib.genK3sAgentModule {
inherit pkgs;
tokenFile = config.age.secrets."k3s-prod-1-token".path;
serverIp = myvars.networking.hostsAddr.${k3sServerName}.ipv4;
# use my own domain & kube-vip's virtual IP for the API server
# so that the API server can always be accessed even if some nodes are down
masterHost = "prod-cluster-1.writefor.fun";
};
in {
imports =

5
hosts/k8s/k3s-prod-1-worker-2/default.nix
View File

@@ -6,7 +6,6 @@
...
}: let
hostName = "k3s-prod-1-worker-2"; # define your hostname.
k3sServerName = "k3s-prod-1-master-1";
coreModule = mylib.genKubeVirtCoreModule {
inherit pkgs hostName;
@@ -15,7 +14,9 @@
k3sModule = mylib.genK3sAgentModule {
inherit pkgs;
tokenFile = config.age.secrets."k3s-prod-1-token".path;
serverIp = myvars.networking.hostsAddr.${k3sServerName}.ipv4;
# use my own domain & kube-vip's virtual IP for the API server
# so that the API server can always be accessed even if some nodes are down
masterHost = "prod-cluster-1.writefor.fun";
};
in {
imports =

5
hosts/k8s/k3s-prod-1-worker-3/default.nix
View File

@@ -6,7 +6,6 @@
...
}: let
hostName = "k3s-prod-1-worker-3"; # define your hostname.
k3sServerName = "k3s-prod-1-master-1";
coreModule = mylib.genKubeVirtCoreModule {
inherit pkgs hostName;
@@ -15,7 +14,9 @@
k3sModule = mylib.genK3sAgentModule {
inherit pkgs;
tokenFile = config.age.secrets."k3s-prod-1-token".path;
serverIp = myvars.networking.hostsAddr.${k3sServerName}.ipv4;
# use my own domain & kube-vip's virtual IP for the API server
# so that the API server can always be accessed even if some nodes are down
masterHost = "prod-cluster-1.writefor.fun";
};
in {
imports =

3
hosts/k8s/k3s-test-1-master-1/default.nix
View File

@@ -17,6 +17,9 @@
tokenFile = config.age.secrets."k3s-prod-1-token".path;
# the first node in the cluster should be the one to initialize the cluster
clusterInit = true;
# use my own domain & kube-vip's virtual IP for the API server
# so that the API server can always be accessed even if some nodes are down
masterHost = "test-cluster-1.writefor.fun";
};
in {
imports =

5
hosts/k8s/k3s-test-1-master-2/default.nix
View File

@@ -6,7 +6,6 @@
...
}: let
hostName = "k3s-test-1-master-2"; # define your hostname.
k3sServerName = "k3s-test-1-master-1";
coreModule = mylib.genKubeVirtCoreModule {
inherit pkgs hostName;
@@ -16,7 +15,9 @@
inherit pkgs;
kubeconfigFile = "/home/${myvars.username}/.kube/config";
tokenFile = config.age.secrets."k3s-prod-1-token".path;
serverIp = myvars.networking.hostsAddr.${k3sServerName}.ipv4;
# use my own domain & kube-vip's virtual IP for the API server
# so that the API server can always be accessed even if some nodes are down
masterHost = "test-cluster-1.writefor.fun";
};
in {
imports =

5
hosts/k8s/k3s-test-1-master-3/default.nix
View File

@@ -6,7 +6,6 @@
...
}: let
hostName = "k3s-test-1-master-3"; # define your hostname.
k3sServerName = "k3s-test-1-master-1";
coreModule = mylib.genKubeVirtCoreModule {
inherit pkgs hostName;
@@ -16,7 +15,9 @@
inherit pkgs;
kubeconfigFile = "/home/${myvars.username}/.kube/config";
tokenFile = config.age.secrets."k3s-prod-1-token".path;
serverIp = myvars.networking.hostsAddr.${k3sServerName}.ipv4;
# use my own domain & kube-vip's virtual IP for the API server
# so that the API server can always be accessed even if some nodes are down
masterHost = "test-cluster-1.writefor.fun";
};
in {
imports =

3
hosts/k8s/kubevirt-shoryu/default.nix
View File

@@ -19,6 +19,9 @@
tokenFile = "/run/media/nixos_k3s/kubevirt-k3s-token";
# the first node in the cluster should be the one to initialize the cluster
clusterInit = true;
# use my own domain & kube-vip's virtual IP for the API server
# so that the API server can always be accessed even if some nodes are down
masterHost = "kubevirt-cluster-1.writefor.fun";
};
in {
imports =

4
hosts/k8s/kubevirt-shushou/default.nix
View File

@@ -17,7 +17,9 @@
inherit pkgs;
kubeconfigFile = "/home/${myvars.username}/.kube/config";
tokenFile = "/run/media/nixos_k3s/kubevirt-k3s-token";
serverIp = myvars.networking.hostsAddr.${k3sServerName}.ipv4;
# use my own domain & kube-vip's virtual IP for the API server
# so that the API server can always be accessed even if some nodes are down
masterHost = "kubevirt-cluster-1.writefor.fun";
};
in {
imports =

4
hosts/k8s/kubevirt-youko/default.nix
View File

@@ -17,7 +17,9 @@
inherit pkgs;
kubeconfigFile = "/home/${myvars.username}/.kube/config";
tokenFile = "/run/media/nixos_k3s/kubevirt-k3s-token";
serverIp = myvars.networking.hostsAddr.${k3sServerName}.ipv4;
# use my own domain & kube-vip's virtual IP for the API server
# so that the API server can always be accessed even if some nodes are down
masterHost = "kubevirt-cluster-1.writefor.fun";
};
in {
imports =

4
lib/genK3sAgentModule.nix
View File

@@ -1,6 +1,6 @@
{
pkgs,
serverIp,
masterHost,
tokenFile,
...
}: let
@@ -12,7 +12,7 @@ in {
inherit package tokenFile;
role = "agent";
serverAddr = "https://${serverIp}:6443";
serverAddr = "https://${masterHost}:6443";
# https://docs.k3s.io/cli/agent
extraFlags = let
flagList = [

11
lib/genK3sServerModule.nix
View File

@@ -6,8 +6,10 @@
# If you are configuring an HA cluster with an embedded etcd,
# the 1st server must have `clusterInit = true`
# and other servers must connect to it using `serverAddr`.
serverIp ? null,
clusterInit ? (serverIp == null),
#
# this can be a domain name or an IP address(such as kube-vip's virtual IP)
masterHost,
clusterInit ? false,
addTaints ? false,
...
}: let
@@ -32,14 +34,14 @@ in {
serverAddr =
if clusterInit
then ""
else "https://${serverIp}:6443";
else "https://${masterHost}:6443";
role = "server";
# https://docs.k3s.io/cli/server
extraFlags = let
flagList =
[
"--write-kubeconfig ${kubeconfigFile}"
"--write-kubeconfig=${kubeconfigFile}"
"--write-kubeconfig-mode=644"
"--service-node-port-range=80-32767"
"--kube-apiserver-arg='--allow-privileged=true'" # required by kubevirt
@@ -52,6 +54,7 @@ in {
"--disable=servicelb" # we use kube-vip instead
"--flannel-backend=none" # we use cilium instead
"--disable-network-policy"
"--tls-san=${masterHost}"
]
# prevent workloads from running on the master
++ (pkgs.lib.optionals addTaints ["--node-taint=CriticalAddonsOnly=true:NoExecute"]);