From e5b95affa64b952a69c3560c52f3b1fc62e5f516 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?ryan4yin=20=7C=20=E4=BA=8C=E8=8A=B1?= Date: Sat, 26 Sep 2026 14:02:03 +0800 Subject: [PATCH] feat(nixos): default-deny host inbound, trust the LAN/tailnet (#325) Hosts used to trust the router as the only boundary (servers ran with the firewall off; service ports were opened globally), so any LAN device could reach every host service. Move filtering to the hosts as defence in depth behind the router, which keeps its firewall on and uses a port mapping for the few ports we actually need. - base: nftables backend; deny inbound by default; trust the home LAN by source and the tailnet by interface (restrict the tailnet with ACLs); suppress per-service `openFirewall` global ports; allow node_exporter only from the monitoring host. The local podman bridge, the tailnet and the pod CIDRs are accepted only after the node_exporter drop. - servers: stop disabling the firewall; drop redundant global port opens (caddy 80/443, NFS 2049, Jellyfin, localsend/test ports); delete the redundant desktop firewall module; aarch64/riscv64 import the firewall explicitly (akane was missing it). - k3s: Cilium pods (10.0.0.0/8, fd00::/8) may reach host services; drop the flannel-era cni0/flannel.1 trust. - youko: postgres listens on localhost + the LAN with pg_hba limited to the LAN; the NFS export stays LAN-scoped; remove the unauthenticated v2ray proxy. - tests: `security-firewall` asserts every host runs the shared firewall, has the LAN CIDR rule and no global ports. --- hosts/12kingdoms-shoryu/service-user-ids.nix | 17 ++++ hosts/12kingdoms-shushou/service-user-ids.nix | 17 ++++ .../homelab-services/README.md | 1 - .../homelab-services/caddy.nix | 6 +- .../homelab-services/jellyfin.nix | 7 +- .../monitoring/victoriametrics.nix | 16 ---- .../homelab-services/postgresql.nix | 4 +- .../homelab-services/proxy.nix | 91 ------------------- hosts/12kingdoms-youko/nfs-golden.nix | 10 +- lib/genK3sAgentModule.nix | 8 ++ lib/genK3sServerModule.nix | 9 ++ modules/nixos/base/networking/firewall.nix | 54 +++++++---- modules/nixos/desktop/networking/firewall.nix | 10 -- modules/nixos/server/server-aarch64.nix | 8 +- modules/nixos/server/server-riscv64.nix | 4 + modules/nixos/server/server.nix | 8 +- .../tests/security-firewall/expected.nix | 4 +- .../tests/security-firewall/expr.nix | 13 ++- .../security-aquamarine-metrics/expected.nix | 1 - .../security-aquamarine-metrics/expr.nix | 1 - .../tests/security-firewall/expected.nix | 4 +- .../tests/security-firewall/expr.nix | 13 ++- vars/networking.nix | 2 + 23 files changed, 139 insertions(+), 169 deletions(-) create mode 100644 hosts/12kingdoms-shoryu/service-user-ids.nix create mode 100644 hosts/12kingdoms-shushou/service-user-ids.nix delete mode 100644 hosts/12kingdoms-youko/homelab-services/proxy.nix delete mode 100644 modules/nixos/desktop/networking/firewall.nix diff --git a/hosts/12kingdoms-shoryu/service-user-ids.nix b/hosts/12kingdoms-shoryu/service-user-ids.nix new file mode 100644 index 00000000..a826d310 --- /dev/null +++ b/hosts/12kingdoms-shoryu/service-user-ids.nix @@ -0,0 +1,17 @@ +{ + ... +}: +{ + # Service users/groups are created by their modules with a dynamically + # allocated uid/gid. That allocation is not stable when the set of system + # users changes, while files on the persistent volumes keep the old numeric + # owner. + # + # microvm runs the k3s guest VMs and owns their persistent state images + # (/var/lib/microvms//{etc,var,home}.img). After its uid shifted, qemu + # could no longer open the images and the guests failed to start on boot with + # "Could not open 'etc.img': Permission denied". Pin it to the value the + # on-disk images use. The ids are host-specific, so this lives in the host + # config rather than a shared module. + users.users.microvm.uid = 997; +} diff --git a/hosts/12kingdoms-shushou/service-user-ids.nix b/hosts/12kingdoms-shushou/service-user-ids.nix new file mode 100644 index 00000000..a9cf0f47 --- /dev/null +++ b/hosts/12kingdoms-shushou/service-user-ids.nix @@ -0,0 +1,17 @@ +{ + ... +}: +{ + # Service users/groups are created by their modules with a dynamically + # allocated uid/gid. That allocation is not stable when the set of system + # users changes, while files on the persistent volumes keep the old numeric + # owner. + # + # microvm runs the k3s guest VMs and owns their persistent state images + # (/var/lib/microvms//{etc,var,home}.img). After its uid shifted, qemu + # could no longer open the images and the guests failed to start on boot with + # "Could not open 'etc.img': Permission denied". Pin it to the value the + # on-disk images use. The ids are host-specific, so this lives in the host + # config rather than a shared module. + users.users.microvm.uid = 992; +} diff --git a/hosts/12kingdoms-youko/homelab-services/README.md b/hosts/12kingdoms-youko/homelab-services/README.md index 2916b72a..29fad387 100644 --- a/hosts/12kingdoms-youko/homelab-services/README.md +++ b/hosts/12kingdoms-youko/homelab-services/README.md @@ -16,7 +16,6 @@ Services: 1. sftpgo: SFTP / WebDAV server. 1. transmission: BitTorrent client. 1. postgresql: Database for the homelab services. -1. v2ray: HTTP/SOCKS proxy for the homelab. 1. restic: Encrypted backups; hosts the REST server the desktops push to. 1. victoriametrics + vmalert + alertmanager + grafana: Monitor the metrics of my homelab. 1. homepage + uptime-kuma: Service dashboard and uptime checks. diff --git a/hosts/12kingdoms-youko/homelab-services/caddy.nix b/hosts/12kingdoms-youko/homelab-services/caddy.nix index 13f31049..02a08cd8 100644 --- a/hosts/12kingdoms-youko/homelab-services/caddy.nix +++ b/hosts/12kingdoms-youko/homelab-services/caddy.nix @@ -239,10 +239,8 @@ in after = [ "acme-writefor.fun.service" ]; }; - networking.firewall.allowedTCPPorts = [ - 80 - 443 - ]; + # caddy is reached from the LAN/tailnet only; the shared firewall + # (modules/nixos/base/networking/firewall.nix) already trusts those. # Serve the wallpapers from the file server. `mkdir -p` here covers the # whole fileserver tree; caddy's dataDir itself is created by the module. diff --git a/hosts/12kingdoms-youko/homelab-services/jellyfin.nix b/hosts/12kingdoms-youko/homelab-services/jellyfin.nix index a35fd14d..c60bf8fd 100644 --- a/hosts/12kingdoms-youko/homelab-services/jellyfin.nix +++ b/hosts/12kingdoms-youko/homelab-services/jellyfin.nix @@ -90,12 +90,7 @@ in ''; # Jellyfin apps discover the server over the LAN via UDP 7359, then talk to - # TCP 8096 directly. Allow both only from the home network; the public entry - # point stays the Caddy vhost on 443. - networking.firewall.extraInputRules = '' - ip saddr 192.168.5.0/24 tcp dport 8096 accept - ip saddr 192.168.5.0/24 udp dport 7359 accept - ''; + # TCP 8096 directly; the shared firewall already trusts the whole LAN. # The library is a separate, `nofail` mount; without this the unit can start # before it is mounted. The module's own RequiresMountsFor entries for diff --git a/hosts/12kingdoms-youko/homelab-services/monitoring/victoriametrics.nix b/hosts/12kingdoms-youko/homelab-services/monitoring/victoriametrics.nix index 1c133ff0..72d41826 100644 --- a/hosts/12kingdoms-youko/homelab-services/monitoring/victoriametrics.nix +++ b/hosts/12kingdoms-youko/homelab-services/monitoring/victoriametrics.nix @@ -93,22 +93,6 @@ in } ]; } - { - job_name = "v2ray-exporter"; - scrape_interval = "30s"; - metrics_path = "/metrics"; - static_configs = [ - { - # same-host exporter bound to loopback (127.0.0.1:9153) - targets = [ "127.0.0.1:9153" ]; - labels.type = "app"; - labels.app = "v2ray"; - labels.host = "aquamarine"; - labels.env = "homelab"; - labels.cluster = "homelab"; - } - ]; - } { job_name = "nut-exporter-homelab-ups"; scrape_interval = "30s"; diff --git a/hosts/12kingdoms-youko/homelab-services/postgresql.nix b/hosts/12kingdoms-youko/homelab-services/postgresql.nix index d3961afc..3cd1900e 100644 --- a/hosts/12kingdoms-youko/homelab-services/postgresql.nix +++ b/hosts/12kingdoms-youko/homelab-services/postgresql.nix @@ -50,6 +50,8 @@ in # https://www.postgresql.org/docs/16/runtime-config.html settings = { port = 5432; + # LAN-only: listen on localhost and the LAN address, never 0.0.0.0. + listen_addresses = lib.mkForce "localhost,${myvars.networking.hostsAddr.youko.ipv4}"; # connections max_connections = 100; @@ -105,7 +107,7 @@ in host replication all ::1/128 trust # Other Remote Access - allow access only the database with the same name as the user - host sameuser all 0.0.0.0/0 scram-sha-256 + host sameuser all ${myvars.networking.lanCidr} scram-sha-256 ''; # initialScript = # pkgs.writeText "backend-initScript" '' diff --git a/hosts/12kingdoms-youko/homelab-services/proxy.nix b/hosts/12kingdoms-youko/homelab-services/proxy.nix deleted file mode 100644 index 3bab4087..00000000 --- a/hosts/12kingdoms-youko/homelab-services/proxy.nix +++ /dev/null @@ -1,91 +0,0 @@ -{ - # dae(running on aquamarine) do not provides http/socks5 proxy server; so we use v2ray here. - # https://github.com/v2fly - services.v2ray = { - enable = true; - config = { - # for monitoring - "stats" = { }; - "api" = { - "tag" = "api"; - "services" = [ - "StatsService" - ]; - }; - "policy" = { - "levels" = { - "0" = { - "statsUserUplink" = true; - "statsUserDownlink" = true; - }; - }; - "system" = { - "statsInboundUplink" = true; - "statsInboundDownlink" = true; - "statsOutboundUplink" = true; - "statsOutboundDownlink" = true; - }; - }; - - inbounds = [ - # core inbound - { - listen = "0.0.0.0"; - port = 7890; - protocol = "http"; - } - { - listen = "0.0.0.0"; - port = 7891; - protocol = "socks"; - settings = { - auth = "noauth"; - udp = true; - }; - } - - # for monitoring - { - "tag" = "api"; - "listen" = "127.0.0.1"; - "port" = 54321; - "protocol" = "dokodemo-door"; - "settings" = { - "address" = "127.0.0.1"; - }; - } - ]; - outbounds = [ - # forward traffic directly via system's default network(to dae proxy running on aquamarine) - { - protocol = "freedom"; - tag = "freedom"; - } - ]; - - # for monitoring - "routing" = { - "rules" = [ - { - "inboundTag" = [ - "api" - ]; - "outboundTag" = "api"; - "type" = "field"; - } - ]; - }; - }; - }; - - # https://github.com/NixOS/nixpkgs/blob/nixos-26.05/nixos/modules/services/monitoring/prometheus/exporters/v2ray.nix - # https://github.com/wi1dcard/v2ray-exporter - services.prometheus.exporters.v2ray = { - enable = true; - # loopback only: scraped by VictoriaMetrics on the same host - listenAddress = "127.0.0.1"; - port = 9153; - openFirewall = false; - v2rayEndpoint = "127.0.0.1:54321"; - }; -} diff --git a/hosts/12kingdoms-youko/nfs-golden.nix b/hosts/12kingdoms-youko/nfs-golden.nix index 986bc621..2ec4cc9a 100644 --- a/hosts/12kingdoms-youko/nfs-golden.nix +++ b/hosts/12kingdoms-youko/nfs-golden.nix @@ -10,8 +10,9 @@ let # free space on this host. goldenDir = "/persistent/nfs/golden"; - # Only the homelab LAN may mount the export. The VM hosts are the - # clients (the CSI node plugin mounts from the node, not from the pod). + # Only the homelab LAN may mount the export; the VM hosts and the k3s nodes + # (which run the NFS-CSI node plugin) are all on it. The store is not + # sensitive, so the whole LAN is fine rather than an explicit client list. clientCidr = "192.168.5.0/24"; in { @@ -25,7 +26,6 @@ in ${goldenDir} ${clientCidr}(rw,sync,no_subtree_check,fsid=0,no_root_squash) ''; - # NFSv4 only needs 2049/tcp (no rpcbind/mountd/statd), so the firewall stays - # minimal. The export itself is already restricted to the LAN CIDR above. - networking.firewall.allowedTCPPorts = [ 2049 ]; + # NFSv4 only needs 2049/tcp (no rpcbind/mountd/statd). The export is limited + # to the LAN, and the shared firewall trusts the LAN. } diff --git a/lib/genK3sAgentModule.nix b/lib/genK3sAgentModule.nix index a94b03da..af225501 100644 --- a/lib/genK3sAgentModule.nix +++ b/lib/genK3sAgentModule.nix @@ -25,6 +25,14 @@ in enableIPv6 = true; }; + # Cilium pods reach host services (e.g. the API server) from the pod CIDR. + # `mkAfter` keeps this after the shared base rules (incl. the node_exporter + # drop), so pods cannot scrape node_exporter either. + networking.firewall.extraInputRules = pkgs.lib.mkAfter '' + ip saddr 10.0.0.0/8 accept + ip6 saddr fd00::/104 accept + ''; + services.k3s = { enable = true; inherit package tokenFile; diff --git a/lib/genK3sServerModule.nix b/lib/genK3sServerModule.nix index f68f0712..3d1856e9 100644 --- a/lib/genK3sServerModule.nix +++ b/lib/genK3sServerModule.nix @@ -51,6 +51,15 @@ in enable = true; enableIPv6 = true; }; + + # Cilium pods reach host services (e.g. the API server) from the pod CIDR. + # `mkAfter` keeps this after the shared base rules (incl. the node_exporter + # drop), so pods cannot scrape node_exporter either. + networking.firewall.extraInputRules = lib.mkAfter '' + ip saddr 10.0.0.0/8 accept + ip6 saddr fd00::/104 accept + ''; + services.k3s = { enable = true; inherit package tokenFile clusterInit; diff --git a/modules/nixos/base/networking/firewall.nix b/modules/nixos/base/networking/firewall.nix index 0673fb60..879aceb2 100644 --- a/modules/nixos/base/networking/firewall.nix +++ b/modules/nixos/base/networking/firewall.nix @@ -1,20 +1,40 @@ +{ lib, myvars, ... }: +let + inherit (myvars.networking) lanCidr; + # The host that scrapes the exporters (VictoriaMetrics runs on youko). + monitoringHost = myvars.networking.hostsAddr.youko.ipv4; +in { - # for security reasons, only open the following ports to the network by default. - networking.firewall.allowedTCPPorts = [ - # localsend - 53317 + # Hosts bring their own firewall as defence in depth behind the router's + # (which stays on, with port mappings for the few ports we need): the home + # LAN and the tailnet are trusted equally (tighten the tailnet with Tailscale + # ACLs), and everything else is denied on both address families. + networking.nftables.enable = true; - # tcp ports for testing & sharing - 63080 - 63081 - 63082 - 63083 - 63084 - 63085 - 63086 - 63087 - 63088 - 63089 - ]; - # networking.firewall.allowedUDPPorts = [ ... ]; + networking.firewall = { + enable = true; + + # Nothing is opened to the Internet. LAN/tailnet/local-container access + # comes from the rules below; per-service `openFirewall` ports are + # suppressed. + allowedTCPPorts = lib.mkForce [ ]; + allowedUDPPorts = lib.mkForce [ ]; + + extraInputRules = '' + # node_exporter is scraped only by the monitoring host, not the whole + # LAN/tailnet/local containers. + ip saddr != ${monitoringHost} tcp dport 9100 drop + + # The local container bridge and the tailnet have the same access as the + # LAN. Kept after the drop above so neither can reach node_exporter. + iifname "podman0" accept + iifname "tailscale0" accept + + # The home LAN is the trusted core. + ip saddr ${lanCidr} accept + ip6 saddr fe80::/10 accept + # Tailscale's WireGuard port, for direct (non-DERP) connections. + udp dport 41641 accept + ''; + }; } diff --git a/modules/nixos/desktop/networking/firewall.nix b/modules/nixos/desktop/networking/firewall.nix deleted file mode 100644 index 601726f4..00000000 --- a/modules/nixos/desktop/networking/firewall.nix +++ /dev/null @@ -1,10 +0,0 @@ -# Desktops hold personal data and laptops join untrusted networks, so enable the -# firewall here (repo-wide default is off, see modules/nixos/base/ssh.nix). -# SSH/Tailscale/LocalSend/Sunshine keep working via their own `openFirewall` options. -{ - networking.firewall = { - enable = true; - # Trust everything from the tailnet. - trustedInterfaces = [ "tailscale0" ]; - }; -} diff --git a/modules/nixos/server/server-aarch64.nix b/modules/nixos/server/server-aarch64.nix index 7ac802cb..bbc9c0ae 100644 --- a/modules/nixos/server/server-aarch64.nix +++ b/modules/nixos/server/server-aarch64.nix @@ -15,16 +15,16 @@ ../base/ssh.nix ../base/user-group.nix + # This server imports base files individually instead of the whole + # modules/nixos/base group, so pull the shared firewall in explicitly. + ../base/networking/firewall.nix + ../../base # AppArmor is wired via modules/nixos/base/default.nix for other hosts; this # aarch64 server imports base files individually, so add it explicitly. (mylib.relativeToRoot "hardening/apparmor") ]; - # Servers run on the trusted internal LAN (NAT'd; WAN protected at the router). - # Keep the firewall off here; the secure default is ON (see modules/nixos/base/ssh.nix). - networking.firewall.enable = false; - # Fix: jasper is marked as broken, refusing to evaluate. environment.enableAllTerminfo = lib.mkForce false; } diff --git a/modules/nixos/server/server-riscv64.nix b/modules/nixos/server/server-riscv64.nix index eb433fab..cc117b77 100644 --- a/modules/nixos/server/server-riscv64.nix +++ b/modules/nixos/server/server-riscv64.nix @@ -13,6 +13,10 @@ ../base/ssh.nix ../base/user-group.nix + # This server imports base files individually instead of the whole + # modules/nixos/base group, so pull the shared firewall in explicitly. + ../base/networking/firewall.nix + ../../base ]; } diff --git a/modules/nixos/server/server.nix b/modules/nixos/server/server.nix index 3e95d887..c9434748 100644 --- a/modules/nixos/server/server.nix +++ b/modules/nixos/server/server.nix @@ -1,11 +1,11 @@ -{ lib, ... }: +{ ... }: { imports = [ ../base ../../base ]; - # Servers run on the trusted internal LAN (NAT'd; WAN protected at the router). - # Keep the firewall off here; the secure default is ON (see modules/nixos/base/ssh.nix). - networking.firewall.enable = false; + # Servers used to keep the firewall off, trusting the router alone. They now + # run the shared base firewall (modules/nixos/base/networking/firewall.nix) + # like every other host, as defence in depth behind the router. } diff --git a/outputs/aarch64-linux/tests/security-firewall/expected.nix b/outputs/aarch64-linux/tests/security-firewall/expected.nix index 6ca2c34e..81b3fbb5 100644 --- a/outputs/aarch64-linux/tests/security-firewall/expected.nix +++ b/outputs/aarch64-linux/tests/security-firewall/expected.nix @@ -2,6 +2,4 @@ lib, outputs, }: -lib.genAttrs (builtins.attrNames outputs.nixosConfigurations) ( - name: name == "ai-niri" || name == "shoukei-niri" -) +lib.genAttrs (builtins.attrNames outputs.nixosConfigurations) (_: true) diff --git a/outputs/aarch64-linux/tests/security-firewall/expr.nix b/outputs/aarch64-linux/tests/security-firewall/expr.nix index 39873f63..483dcb80 100644 --- a/outputs/aarch64-linux/tests/security-firewall/expr.nix +++ b/outputs/aarch64-linux/tests/security-firewall/expr.nix @@ -1,7 +1,18 @@ { + myvars, lib, outputs, }: lib.genAttrs (builtins.attrNames outputs.nixosConfigurations) ( - name: outputs.nixosConfigurations.${name}.config.networking.firewall.enable + name: + let + firewall = outputs.nixosConfigurations.${name}.config.networking.firewall; + in + # Every host must run the shared firewall (modules/nixos/base/networking/ + # firewall.nix): enabled, trusting the LAN by source, and opening nothing to + # the Internet through the global port lists. + firewall.enable + && lib.hasInfix myvars.networking.lanCidr firewall.extraInputRules + && firewall.allowedTCPPorts == [ ] + && firewall.allowedUDPPorts == [ ] ) diff --git a/outputs/x86_64-linux/tests/security-aquamarine-metrics/expected.nix b/outputs/x86_64-linux/tests/security-aquamarine-metrics/expected.nix index c45419fd..68785c7f 100644 --- a/outputs/x86_64-linux/tests/security-aquamarine-metrics/expected.nix +++ b/outputs/x86_64-linux/tests/security-aquamarine-metrics/expected.nix @@ -3,7 +3,6 @@ outputs, }: { - v2rayExporter = "127.0.0.1"; pgExporter = "127.0.0.1"; sftpgoTelemetry = "127.0.0.1"; } diff --git a/outputs/x86_64-linux/tests/security-aquamarine-metrics/expr.nix b/outputs/x86_64-linux/tests/security-aquamarine-metrics/expr.nix index ad81b217..65e0e966 100644 --- a/outputs/x86_64-linux/tests/security-aquamarine-metrics/expr.nix +++ b/outputs/x86_64-linux/tests/security-aquamarine-metrics/expr.nix @@ -6,7 +6,6 @@ let aqua = outputs.nixosConfigurations.youko.config; in { - v2rayExporter = aqua.services.prometheus.exporters.v2ray.listenAddress; pgExporter = aqua.services.prometheus.exporters.postgres.listenAddress; sftpgoTelemetry = aqua.services.sftpgo.settings.telemetry.bind_address; } diff --git a/outputs/x86_64-linux/tests/security-firewall/expected.nix b/outputs/x86_64-linux/tests/security-firewall/expected.nix index 6ca2c34e..81b3fbb5 100644 --- a/outputs/x86_64-linux/tests/security-firewall/expected.nix +++ b/outputs/x86_64-linux/tests/security-firewall/expected.nix @@ -2,6 +2,4 @@ lib, outputs, }: -lib.genAttrs (builtins.attrNames outputs.nixosConfigurations) ( - name: name == "ai-niri" || name == "shoukei-niri" -) +lib.genAttrs (builtins.attrNames outputs.nixosConfigurations) (_: true) diff --git a/outputs/x86_64-linux/tests/security-firewall/expr.nix b/outputs/x86_64-linux/tests/security-firewall/expr.nix index 39873f63..483dcb80 100644 --- a/outputs/x86_64-linux/tests/security-firewall/expr.nix +++ b/outputs/x86_64-linux/tests/security-firewall/expr.nix @@ -1,7 +1,18 @@ { + myvars, lib, outputs, }: lib.genAttrs (builtins.attrNames outputs.nixosConfigurations) ( - name: outputs.nixosConfigurations.${name}.config.networking.firewall.enable + name: + let + firewall = outputs.nixosConfigurations.${name}.config.networking.firewall; + in + # Every host must run the shared firewall (modules/nixos/base/networking/ + # firewall.nix): enabled, trusting the LAN by source, and opening nothing to + # the Internet through the global port lists. + firewall.enable + && lib.hasInfix myvars.networking.lanCidr firewall.extraInputRules + && firewall.allowedTCPPorts == [ ] + && firewall.allowedUDPPorts == [ ] ) diff --git a/vars/networking.nix b/vars/networking.nix index 3ef578ae..be4647e5 100644 --- a/vars/networking.nix +++ b/vars/networking.nix @@ -2,6 +2,8 @@ rec { mainGateway = "192.168.5.1"; # main router mainGateway6 = "fe80::5"; # main router's link-local address + # The trusted home LAN; the shared host firewall trusts this by source. + lanCidr = "192.168.5.0/24"; # use suzi as the default gateway # it's a subrouter with a transparent proxy proxyGateway = "192.168.5.178";