{ pkgs, kubeconfigFile, tokenFile, # Initialize HA cluster using an embedded etcd datastore. # If you are configuring an HA cluster with an embedded etcd, # the 1st server must have `clusterInit = true` # and other servers must connect to it using `serverAddr`. # # this can be a domain name or an IP address(such as kube-vip's virtual IP) masterHost, clusterInit ? false, kubeletExtraArgs ? [ ], k3sExtraArgs ? [ ], # Bind embedded K3s control-plane HTTPS metrics to the node address so vmagent can scrape them. metricsBindAddress ? null, nodeLabels ? [ ], nodeTaints ? [ ], disableFlannel ? true, ... }: let lib = pkgs.lib; package = pkgs.k3s; in { environment.systemPackages = with pkgs; [ package k9s kubectl istioctl kubernetes-helm cilium-cli fluxcd clusterctl # for kubernetes cluster-api skopeo # copy/sync images between registries and local storage go-containerregistry # provides `crane` & `gcrane`, it's similar to skopeo dive # explore docker layers ]; networking.enableIPv6 = true; boot.kernel.sysctl = { "net.ipv4.ip_forward" = 1; "net.ipv6.conf.all.forwarding" = 1; }; # Deny pods the credential-bearing host ports before the pod accept below. networking.firewall.extraInputRules = lib.mkAfter '' ip saddr 10.0.0.0/8 tcp dport { 22, 2049, 2283, 2379, 5432, 5900, 5901, 5902, 5903, 8000, 8081, 8082, 9090, 9093, 9100, 9633, 9835 } drop ip6 saddr fd00::/104 tcp dport { 22, 2049, 2283, 2379, 5432, 5900, 5901, 5902, 5903, 8000, 8081, 8082, 9090, 9093, 9100, 9633, 9835 } drop ip saddr 10.0.0.0/8 accept ip6 saddr fd00::/104 accept ''; services.k3s = { enable = true; inherit package tokenFile clusterInit; serverAddr = if clusterInit then "" else "https://${masterHost}:6443"; role = "server"; # https://docs.k3s.io/cli/server extraFlags = let flagList = [ "--write-kubeconfig=${kubeconfigFile}" "--write-kubeconfig-mode=600" # The Istio ingress Service uses NodePorts 80 and 443. "--service-node-port-range=80-32767" # The NFS CSI node plugin runs a privileged container to mount volumes. "--kube-apiserver-arg='--allow-privileged=true'" "--data-dir /var/lib/rancher/k3s" "--etcd-expose-metrics=true" "--etcd-snapshot-schedule-cron='0 */12 * * *'" # disable some features we don't need "--disable-helm-controller" # we use fluxcd instead "--disable=traefik" # deploy our own ingress controller instead "--disable=servicelb" # we use kube-vip instead "--disable-network-policy" # Cilium enforces network policies instead "--disable-kube-proxy" # Cilium's eBPF replacement handles services "--tls-san=${masterHost}" ] ++ (map (label: "--node-label=${label}") nodeLabels) ++ (map (taint: "--node-taint=${taint}") nodeTaints) ++ (map (arg: "--kubelet-arg=${arg}") kubeletExtraArgs) ++ (lib.optionals (metricsBindAddress != null) [ "--kube-controller-manager-arg=bind-address=${metricsBindAddress}" "--kube-scheduler-arg=bind-address=${metricsBindAddress}" ]) ++ (lib.optionals disableFlannel [ "--flannel-backend=none" ]) ++ k3sExtraArgs; in lib.concatStringsSep " " flagList; }; # create symlinks to link k3s's cni directory to the one used by almost all CNI plugins # such as multus, calico, etc. # https://www.freedesktop.org/software/systemd/man/latest/tmpfiles.d.html#Type systemd.tmpfiles.rules = [ # https://docs.k3s.io/networking/multus-ipams "L+ /opt/cni/bin - - - - /var/lib/rancher/k3s/data/cni/" # If you have disabled flannel, you will have to create the directory via a tmpfiles rule "d /var/lib/rancher/k3s/agent/etc/cni/net.d 0751 root root - -" # Link the CNI config directory "L+ /etc/cni/net.d - - - - /var/lib/rancher/k3s/agent/etc/cni/net.d" ]; }