From df6f9b8d6d3fd41de6017a6ca9840de585ee0a28 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?ryan4yin=20=7C=20=E4=BA=8C=E8=8A=B1?= Date: Wed, 7 Oct 2026 17:58:33 +0800 Subject: [PATCH] feat(k3s): disable kube-proxy for Cilium eBPF replacement (#404) --- lib/genK3sAgentModule.nix | 27 +++++---------------------- lib/genK3sServerModule.nix | 22 +++++----------------- 2 files changed, 10 insertions(+), 39 deletions(-) diff --git a/lib/genK3sAgentModule.nix b/lib/genK3sAgentModule.nix index c6729a57..acd7703f 100644 --- a/lib/genK3sAgentModule.nix +++ b/lib/genK3sAgentModule.nix @@ -12,26 +12,13 @@ in { environment.systemPackages = [ package ]; - # Kernel modules required by cilium - boot.kernelModules = [ - "ip6_tables" - "ip6table_mangle" - "ip6table_raw" - "ip6table_filter" - ]; networking.enableIPv6 = true; - networking.nat = { - enable = true; - enableIPv6 = true; + boot.kernel.sysctl = { + "net.ipv4.ip_forward" = 1; + "net.ipv6.conf.all.forwarding" = 1; }; - # Pods may reach the host services they need, but they are not part of the - # host's trusted LAN. Deny the credential-bearing host ports before the broad - # pod-network accept below, so a compromised pod cannot reach NFS, the - # database, the backup server, the VNC consoles, or an exporter. `mkAfter` - # keeps this after the shared base rules (incl. the node_exporter drop). - # Ports 80/443 stay reachable so in-cluster calls through the ingress are not - # broken; the ingress is the authentication boundary there. + # Deny pods the credential-bearing host ports before the pod accept below. networking.firewall.extraInputRules = pkgs.lib.mkAfter '' ip saddr 10.0.0.0/8 tcp dport { 22, 2049, 2283, 5432, 5900, 5901, 5902, 5903, 8000, 8081, 8082, 9090, 9093, 9100, 9633, 9835 } drop ip6 saddr fd00::/104 tcp dport { 22, 2049, 2283, 5432, 5900, 5901, 5902, 5903, 8000, 8081, 8082, 9090, 9093, 9100, 9633, 9835 } drop @@ -57,11 +44,7 @@ in pkgs.lib.concatStringsSep " " flagList; }; - # Link k3s's CNI directories to the conventional locations, so CNI plugins - # that write to /etc/cni/net.d or /opt/cni/bin (cilium, istio-cni) share them - # with k3s. Without this, cilium writes /etc/cni/net.d while istio-cni reads - # k3s's own dir, and the chained plugin never finds a network config. - # Mirrors the server module. + # Share k3s's CNI dirs with plugins using the conventional paths (cilium, istio-cni). systemd.tmpfiles.rules = [ "L+ /opt/cni/bin - - - - /var/lib/rancher/k3s/data/cni/" "d /var/lib/rancher/k3s/agent/etc/cni/net.d 0751 root root - -" diff --git a/lib/genK3sServerModule.nix b/lib/genK3sServerModule.nix index 46f8539f..821e6dc4 100644 --- a/lib/genK3sServerModule.nix +++ b/lib/genK3sServerModule.nix @@ -39,26 +39,13 @@ in dive # explore docker layers ]; - # Kernel modules required by cilium - boot.kernelModules = [ - "ip6_tables" - "ip6table_mangle" - "ip6table_raw" - "ip6table_filter" - ]; networking.enableIPv6 = true; - networking.nat = { - enable = true; - enableIPv6 = true; + boot.kernel.sysctl = { + "net.ipv4.ip_forward" = 1; + "net.ipv6.conf.all.forwarding" = 1; }; - # Pods may reach the host services they need, but they are not part of the - # host's trusted LAN. Deny the credential-bearing host ports before the broad - # pod-network accept below, so a compromised pod cannot reach NFS, the - # database, the backup server, the VNC consoles, or an exporter. `mkAfter` - # keeps this after the shared base rules (incl. the node_exporter drop). - # Ports 80/443 stay reachable so in-cluster calls through the ingress are not - # broken; the ingress is the authentication boundary there. + # Deny pods the credential-bearing host ports before the pod accept below. networking.firewall.extraInputRules = lib.mkAfter '' ip saddr 10.0.0.0/8 tcp dport { 22, 2049, 2283, 5432, 5900, 5901, 5902, 5903, 8000, 8081, 8082, 9090, 9093, 9100, 9633, 9835 } drop ip6 saddr fd00::/104 tcp dport { 22, 2049, 2283, 5432, 5900, 5901, 5902, 5903, 8000, 8081, 8082, 9090, 9093, 9100, 9633, 9835 } drop @@ -90,6 +77,7 @@ in "--disable=traefik" # deploy our own ingress controller instead "--disable=servicelb" # we use kube-vip instead "--disable-network-policy" # Cilium enforces network policies instead + "--disable-kube-proxy" # Cilium's eBPF replacement handles services "--tls-san=${masterHost}" ] ++ (map (label: "--node-label=${label}") nodeLabels)