mirror of
https://github.com/ryan4yin/nix-config.git
synced 2026-11-13 12:13:46 +01:00
Ryan Yin
d4aa25d368
The etcd client API listens on the masters' static LAN address, and the pod-source deny lists omitted it. Add 2379 to both nft drop lines and assert it in the security-k3s-pod-firewall eval test. 2381 (etcd metrics, scraped by in-cluster monitoring) stays reachable by design. Pairs with the matching Cilium egressDeny port in k8s-gitops. Assisted-by: DeepSeek Harness (gpt-6-astra)
108 lines
4.0 KiB
Nix
108 lines
4.0 KiB
Nix
{
|
|
pkgs,
|
|
kubeconfigFile,
|
|
tokenFile,
|
|
# Initialize HA cluster using an embedded etcd datastore.
|
|
# If you are configuring an HA cluster with an embedded etcd,
|
|
# the 1st server must have `clusterInit = true`
|
|
# and other servers must connect to it using `serverAddr`.
|
|
#
|
|
# this can be a domain name or an IP address(such as kube-vip's virtual IP)
|
|
masterHost,
|
|
clusterInit ? false,
|
|
kubeletExtraArgs ? [ ],
|
|
k3sExtraArgs ? [ ],
|
|
# Bind embedded K3s control-plane HTTPS metrics to the node address so vmagent can scrape them.
|
|
metricsBindAddress ? null,
|
|
nodeLabels ? [ ],
|
|
nodeTaints ? [ ],
|
|
disableFlannel ? true,
|
|
...
|
|
}:
|
|
let
|
|
lib = pkgs.lib;
|
|
package = pkgs.k3s;
|
|
in
|
|
{
|
|
environment.systemPackages = with pkgs; [
|
|
package
|
|
k9s
|
|
kubectl
|
|
istioctl
|
|
kubernetes-helm
|
|
cilium-cli
|
|
fluxcd
|
|
clusterctl # for kubernetes cluster-api
|
|
|
|
skopeo # copy/sync images between registries and local storage
|
|
go-containerregistry # provides `crane` & `gcrane`, it's similar to skopeo
|
|
dive # explore docker layers
|
|
];
|
|
|
|
networking.enableIPv6 = true;
|
|
boot.kernel.sysctl = {
|
|
"net.ipv4.ip_forward" = 1;
|
|
"net.ipv6.conf.all.forwarding" = 1;
|
|
};
|
|
|
|
# Deny pods the credential-bearing host ports before the pod accept below.
|
|
networking.firewall.extraInputRules = lib.mkAfter ''
|
|
ip saddr 10.0.0.0/8 tcp dport { 22, 2049, 2283, 2379, 5432, 5900, 5901, 5902, 5903, 8000, 8081, 8082, 9090, 9093, 9100, 9633, 9835 } drop
|
|
ip6 saddr fd00::/104 tcp dport { 22, 2049, 2283, 2379, 5432, 5900, 5901, 5902, 5903, 8000, 8081, 8082, 9090, 9093, 9100, 9633, 9835 } drop
|
|
ip saddr 10.0.0.0/8 accept
|
|
ip6 saddr fd00::/104 accept
|
|
'';
|
|
|
|
services.k3s = {
|
|
enable = true;
|
|
inherit package tokenFile clusterInit;
|
|
serverAddr = if clusterInit then "" else "https://${masterHost}:6443";
|
|
|
|
role = "server";
|
|
# https://docs.k3s.io/cli/server
|
|
extraFlags =
|
|
let
|
|
flagList = [
|
|
"--write-kubeconfig=${kubeconfigFile}"
|
|
"--write-kubeconfig-mode=600"
|
|
# The Istio ingress Service uses NodePorts 80 and 443.
|
|
"--service-node-port-range=80-32767"
|
|
# The NFS CSI node plugin runs a privileged container to mount volumes.
|
|
"--kube-apiserver-arg='--allow-privileged=true'"
|
|
"--data-dir /var/lib/rancher/k3s"
|
|
"--etcd-expose-metrics=true"
|
|
"--etcd-snapshot-schedule-cron='0 */12 * * *'"
|
|
# disable some features we don't need
|
|
"--disable-helm-controller" # we use fluxcd instead
|
|
"--disable=traefik" # deploy our own ingress controller instead
|
|
"--disable=servicelb" # we use kube-vip instead
|
|
"--disable-network-policy" # Cilium enforces network policies instead
|
|
"--disable-kube-proxy" # Cilium's eBPF replacement handles services
|
|
"--tls-san=${masterHost}"
|
|
]
|
|
++ (map (label: "--node-label=${label}") nodeLabels)
|
|
++ (map (taint: "--node-taint=${taint}") nodeTaints)
|
|
++ (map (arg: "--kubelet-arg=${arg}") kubeletExtraArgs)
|
|
++ (lib.optionals (metricsBindAddress != null) [
|
|
"--kube-controller-manager-arg=bind-address=${metricsBindAddress}"
|
|
"--kube-scheduler-arg=bind-address=${metricsBindAddress}"
|
|
])
|
|
++ (lib.optionals disableFlannel [ "--flannel-backend=none" ])
|
|
++ k3sExtraArgs;
|
|
in
|
|
lib.concatStringsSep " " flagList;
|
|
};
|
|
|
|
# create symlinks to link k3s's cni directory to the one used by almost all CNI plugins
|
|
# such as multus, calico, etc.
|
|
# https://www.freedesktop.org/software/systemd/man/latest/tmpfiles.d.html#Type
|
|
systemd.tmpfiles.rules = [
|
|
# https://docs.k3s.io/networking/multus-ipams
|
|
"L+ /opt/cni/bin - - - - /var/lib/rancher/k3s/data/cni/"
|
|
# If you have disabled flannel, you will have to create the directory via a tmpfiles rule
|
|
"d /var/lib/rancher/k3s/agent/etc/cni/net.d 0751 root root - -"
|
|
# Link the CNI config directory
|
|
"L+ /etc/cni/net.d - - - - /var/lib/rancher/k3s/agent/etc/cni/net.d"
|
|
];
|
|
}
|