mirror of
https://github.com/ryan4yin/nix-config.git
synced 2026-11-13 12:13:36 +01:00
ryan4yin | 二花
e5b95affa6
Hosts used to trust the router as the only boundary (servers ran with the firewall off; service ports were opened globally), so any LAN device could reach every host service. Move filtering to the hosts as defence in depth behind the router, which keeps its firewall on and uses a port mapping for the few ports we actually need. - base: nftables backend; deny inbound by default; trust the home LAN by source and the tailnet by interface (restrict the tailnet with ACLs); suppress per-service `openFirewall` global ports; allow node_exporter only from the monitoring host. The local podman bridge, the tailnet and the pod CIDRs are accepted only after the node_exporter drop. - servers: stop disabling the firewall; drop redundant global port opens (caddy 80/443, NFS 2049, Jellyfin, localsend/test ports); delete the redundant desktop firewall module; aarch64/riscv64 import the firewall explicitly (akane was missing it). - k3s: Cilium pods (10.0.0.0/8, fd00::/8) may reach host services; drop the flannel-era cni0/flannel.1 trust. - youko: postgres listens on localhost + the LAN with pg_hba limited to the LAN; the NFS export stays LAN-scoped; remove the unauthenticated v2ray proxy. - tests: `security-firewall` asserts every host runs the shared firewall, has the LAN CIDR rule and no global ports.
114 lines
3.8 KiB
Nix
114 lines
3.8 KiB
Nix
{
|
|
pkgs,
|
|
kubeconfigFile,
|
|
tokenFile,
|
|
# Initialize HA cluster using an embedded etcd datastore.
|
|
# If you are configuring an HA cluster with an embedded etcd,
|
|
# the 1st server must have `clusterInit = true`
|
|
# and other servers must connect to it using `serverAddr`.
|
|
#
|
|
# this can be a domain name or an IP address(such as kube-vip's virtual IP)
|
|
masterHost,
|
|
clusterInit ? false,
|
|
kubeletExtraArgs ? [ ],
|
|
k3sExtraArgs ? [ ],
|
|
# Bind embedded K3s control-plane HTTPS metrics to the node address so vmagent can scrape them.
|
|
metricsBindAddress ? null,
|
|
nodeLabels ? [ ],
|
|
nodeTaints ? [ ],
|
|
disableFlannel ? true,
|
|
...
|
|
}:
|
|
let
|
|
lib = pkgs.lib;
|
|
package = pkgs.k3s;
|
|
in
|
|
{
|
|
environment.systemPackages = with pkgs; [
|
|
package
|
|
k9s
|
|
kubectl
|
|
istioctl
|
|
kubernetes-helm
|
|
cilium-cli
|
|
fluxcd
|
|
clusterctl # for kubernetes cluster-api
|
|
|
|
skopeo # copy/sync images between registries and local storage
|
|
go-containerregistry # provides `crane` & `gcrane`, it's similar to skopeo
|
|
dive # explore docker layers
|
|
];
|
|
|
|
# Kernel modules required by cilium
|
|
boot.kernelModules = [
|
|
"ip6_tables"
|
|
"ip6table_mangle"
|
|
"ip6table_raw"
|
|
"ip6table_filter"
|
|
];
|
|
networking.enableIPv6 = true;
|
|
networking.nat = {
|
|
enable = true;
|
|
enableIPv6 = true;
|
|
};
|
|
|
|
# Cilium pods reach host services (e.g. the API server) from the pod CIDR.
|
|
# `mkAfter` keeps this after the shared base rules (incl. the node_exporter
|
|
# drop), so pods cannot scrape node_exporter either.
|
|
networking.firewall.extraInputRules = lib.mkAfter ''
|
|
ip saddr 10.0.0.0/8 accept
|
|
ip6 saddr fd00::/104 accept
|
|
'';
|
|
|
|
services.k3s = {
|
|
enable = true;
|
|
inherit package tokenFile clusterInit;
|
|
serverAddr = if clusterInit then "" else "https://${masterHost}:6443";
|
|
|
|
role = "server";
|
|
# https://docs.k3s.io/cli/server
|
|
extraFlags =
|
|
let
|
|
flagList = [
|
|
"--write-kubeconfig=${kubeconfigFile}"
|
|
"--write-kubeconfig-mode=600"
|
|
# The Istio ingress Service uses NodePorts 80 and 443.
|
|
"--service-node-port-range=80-32767"
|
|
# The NFS CSI node plugin runs a privileged container to mount volumes.
|
|
"--kube-apiserver-arg='--allow-privileged=true'"
|
|
"--data-dir /var/lib/rancher/k3s"
|
|
"--etcd-expose-metrics=true"
|
|
"--etcd-snapshot-schedule-cron='0 */12 * * *'"
|
|
# disable some features we don't need
|
|
"--disable-helm-controller" # we use fluxcd instead
|
|
"--disable=traefik" # deploy our own ingress controller instead
|
|
"--disable=servicelb" # we use kube-vip instead
|
|
"--disable-network-policy" # Cilium enforces network policies instead
|
|
"--tls-san=${masterHost}"
|
|
]
|
|
++ (map (label: "--node-label=${label}") nodeLabels)
|
|
++ (map (taint: "--node-taint=${taint}") nodeTaints)
|
|
++ (map (arg: "--kubelet-arg=${arg}") kubeletExtraArgs)
|
|
++ (lib.optionals (metricsBindAddress != null) [
|
|
"--kube-controller-manager-arg=bind-address=${metricsBindAddress}"
|
|
"--kube-scheduler-arg=bind-address=${metricsBindAddress}"
|
|
])
|
|
++ (lib.optionals disableFlannel [ "--flannel-backend=none" ])
|
|
++ k3sExtraArgs;
|
|
in
|
|
lib.concatStringsSep " " flagList;
|
|
};
|
|
|
|
# create symlinks to link k3s's cni directory to the one used by almost all CNI plugins
|
|
# such as multus, calico, etc.
|
|
# https://www.freedesktop.org/software/systemd/man/latest/tmpfiles.d.html#Type
|
|
systemd.tmpfiles.rules = [
|
|
# https://docs.k3s.io/networking/multus-ipams
|
|
"L+ /opt/cni/bin - - - - /var/lib/rancher/k3s/data/cni/"
|
|
# If you have disabled flannel, you will have to create the directory via a tmpfiles rule
|
|
"d /var/lib/rancher/k3s/agent/etc/cni/net.d 0751 root root - -"
|
|
# Link the CNI config directory
|
|
"L+ /etc/cni/net.d - - - - /var/lib/rancher/k3s/agent/etc/cni/net.d"
|
|
];
|
|
}
|