diff --git a/hosts/12kingdoms-shoryu/service-user-ids.nix b/hosts/12kingdoms-shoryu/service-user-ids.nix new file mode 100644 index 00000000..a826d310 --- /dev/null +++ b/hosts/12kingdoms-shoryu/service-user-ids.nix @@ -0,0 +1,17 @@ +{ + ... +}: +{ + # Service users/groups are created by their modules with a dynamically + # allocated uid/gid. That allocation is not stable when the set of system + # users changes, while files on the persistent volumes keep the old numeric + # owner. + # + # microvm runs the k3s guest VMs and owns their persistent state images + # (/var/lib/microvms//{etc,var,home}.img). After its uid shifted, qemu + # could no longer open the images and the guests failed to start on boot with + # "Could not open 'etc.img': Permission denied". Pin it to the value the + # on-disk images use. The ids are host-specific, so this lives in the host + # config rather than a shared module. + users.users.microvm.uid = 997; +} diff --git a/hosts/12kingdoms-shushou/service-user-ids.nix b/hosts/12kingdoms-shushou/service-user-ids.nix new file mode 100644 index 00000000..a9cf0f47 --- /dev/null +++ b/hosts/12kingdoms-shushou/service-user-ids.nix @@ -0,0 +1,17 @@ +{ + ... +}: +{ + # Service users/groups are created by their modules with a dynamically + # allocated uid/gid. That allocation is not stable when the set of system + # users changes, while files on the persistent volumes keep the old numeric + # owner. + # + # microvm runs the k3s guest VMs and owns their persistent state images + # (/var/lib/microvms//{etc,var,home}.img). After its uid shifted, qemu + # could no longer open the images and the guests failed to start on boot with + # "Could not open 'etc.img': Permission denied". Pin it to the value the + # on-disk images use. The ids are host-specific, so this lives in the host + # config rather than a shared module. + users.users.microvm.uid = 992; +} diff --git a/hosts/12kingdoms-youko/homelab-services/README.md b/hosts/12kingdoms-youko/homelab-services/README.md index 2916b72a..29fad387 100644 --- a/hosts/12kingdoms-youko/homelab-services/README.md +++ b/hosts/12kingdoms-youko/homelab-services/README.md @@ -16,7 +16,6 @@ Services: 1. sftpgo: SFTP / WebDAV server. 1. transmission: BitTorrent client. 1. postgresql: Database for the homelab services. -1. v2ray: HTTP/SOCKS proxy for the homelab. 1. restic: Encrypted backups; hosts the REST server the desktops push to. 1. victoriametrics + vmalert + alertmanager + grafana: Monitor the metrics of my homelab. 1. homepage + uptime-kuma: Service dashboard and uptime checks. diff --git a/hosts/12kingdoms-youko/homelab-services/caddy.nix b/hosts/12kingdoms-youko/homelab-services/caddy.nix index 13f31049..02a08cd8 100644 --- a/hosts/12kingdoms-youko/homelab-services/caddy.nix +++ b/hosts/12kingdoms-youko/homelab-services/caddy.nix @@ -239,10 +239,8 @@ in after = [ "acme-writefor.fun.service" ]; }; - networking.firewall.allowedTCPPorts = [ - 80 - 443 - ]; + # caddy is reached from the LAN/tailnet only; the shared firewall + # (modules/nixos/base/networking/firewall.nix) already trusts those. # Serve the wallpapers from the file server. `mkdir -p` here covers the # whole fileserver tree; caddy's dataDir itself is created by the module. diff --git a/hosts/12kingdoms-youko/homelab-services/jellyfin.nix b/hosts/12kingdoms-youko/homelab-services/jellyfin.nix index a35fd14d..c60bf8fd 100644 --- a/hosts/12kingdoms-youko/homelab-services/jellyfin.nix +++ b/hosts/12kingdoms-youko/homelab-services/jellyfin.nix @@ -90,12 +90,7 @@ in ''; # Jellyfin apps discover the server over the LAN via UDP 7359, then talk to - # TCP 8096 directly. Allow both only from the home network; the public entry - # point stays the Caddy vhost on 443. - networking.firewall.extraInputRules = '' - ip saddr 192.168.5.0/24 tcp dport 8096 accept - ip saddr 192.168.5.0/24 udp dport 7359 accept - ''; + # TCP 8096 directly; the shared firewall already trusts the whole LAN. # The library is a separate, `nofail` mount; without this the unit can start # before it is mounted. The module's own RequiresMountsFor entries for diff --git a/hosts/12kingdoms-youko/homelab-services/monitoring/victoriametrics.nix b/hosts/12kingdoms-youko/homelab-services/monitoring/victoriametrics.nix index 1c133ff0..72d41826 100644 --- a/hosts/12kingdoms-youko/homelab-services/monitoring/victoriametrics.nix +++ b/hosts/12kingdoms-youko/homelab-services/monitoring/victoriametrics.nix @@ -93,22 +93,6 @@ in } ]; } - { - job_name = "v2ray-exporter"; - scrape_interval = "30s"; - metrics_path = "/metrics"; - static_configs = [ - { - # same-host exporter bound to loopback (127.0.0.1:9153) - targets = [ "127.0.0.1:9153" ]; - labels.type = "app"; - labels.app = "v2ray"; - labels.host = "aquamarine"; - labels.env = "homelab"; - labels.cluster = "homelab"; - } - ]; - } { job_name = "nut-exporter-homelab-ups"; scrape_interval = "30s"; diff --git a/hosts/12kingdoms-youko/homelab-services/postgresql.nix b/hosts/12kingdoms-youko/homelab-services/postgresql.nix index d3961afc..3cd1900e 100644 --- a/hosts/12kingdoms-youko/homelab-services/postgresql.nix +++ b/hosts/12kingdoms-youko/homelab-services/postgresql.nix @@ -50,6 +50,8 @@ in # https://www.postgresql.org/docs/16/runtime-config.html settings = { port = 5432; + # LAN-only: listen on localhost and the LAN address, never 0.0.0.0. + listen_addresses = lib.mkForce "localhost,${myvars.networking.hostsAddr.youko.ipv4}"; # connections max_connections = 100; @@ -105,7 +107,7 @@ in host replication all ::1/128 trust # Other Remote Access - allow access only the database with the same name as the user - host sameuser all 0.0.0.0/0 scram-sha-256 + host sameuser all ${myvars.networking.lanCidr} scram-sha-256 ''; # initialScript = # pkgs.writeText "backend-initScript" '' diff --git a/hosts/12kingdoms-youko/homelab-services/proxy.nix b/hosts/12kingdoms-youko/homelab-services/proxy.nix deleted file mode 100644 index 3bab4087..00000000 --- a/hosts/12kingdoms-youko/homelab-services/proxy.nix +++ /dev/null @@ -1,91 +0,0 @@ -{ - # dae(running on aquamarine) do not provides http/socks5 proxy server; so we use v2ray here. - # https://github.com/v2fly - services.v2ray = { - enable = true; - config = { - # for monitoring - "stats" = { }; - "api" = { - "tag" = "api"; - "services" = [ - "StatsService" - ]; - }; - "policy" = { - "levels" = { - "0" = { - "statsUserUplink" = true; - "statsUserDownlink" = true; - }; - }; - "system" = { - "statsInboundUplink" = true; - "statsInboundDownlink" = true; - "statsOutboundUplink" = true; - "statsOutboundDownlink" = true; - }; - }; - - inbounds = [ - # core inbound - { - listen = "0.0.0.0"; - port = 7890; - protocol = "http"; - } - { - listen = "0.0.0.0"; - port = 7891; - protocol = "socks"; - settings = { - auth = "noauth"; - udp = true; - }; - } - - # for monitoring - { - "tag" = "api"; - "listen" = "127.0.0.1"; - "port" = 54321; - "protocol" = "dokodemo-door"; - "settings" = { - "address" = "127.0.0.1"; - }; - } - ]; - outbounds = [ - # forward traffic directly via system's default network(to dae proxy running on aquamarine) - { - protocol = "freedom"; - tag = "freedom"; - } - ]; - - # for monitoring - "routing" = { - "rules" = [ - { - "inboundTag" = [ - "api" - ]; - "outboundTag" = "api"; - "type" = "field"; - } - ]; - }; - }; - }; - - # https://github.com/NixOS/nixpkgs/blob/nixos-26.05/nixos/modules/services/monitoring/prometheus/exporters/v2ray.nix - # https://github.com/wi1dcard/v2ray-exporter - services.prometheus.exporters.v2ray = { - enable = true; - # loopback only: scraped by VictoriaMetrics on the same host - listenAddress = "127.0.0.1"; - port = 9153; - openFirewall = false; - v2rayEndpoint = "127.0.0.1:54321"; - }; -} diff --git a/hosts/12kingdoms-youko/nfs-golden.nix b/hosts/12kingdoms-youko/nfs-golden.nix index 986bc621..2ec4cc9a 100644 --- a/hosts/12kingdoms-youko/nfs-golden.nix +++ b/hosts/12kingdoms-youko/nfs-golden.nix @@ -10,8 +10,9 @@ let # free space on this host. goldenDir = "/persistent/nfs/golden"; - # Only the homelab LAN may mount the export. The VM hosts are the - # clients (the CSI node plugin mounts from the node, not from the pod). + # Only the homelab LAN may mount the export; the VM hosts and the k3s nodes + # (which run the NFS-CSI node plugin) are all on it. The store is not + # sensitive, so the whole LAN is fine rather than an explicit client list. clientCidr = "192.168.5.0/24"; in { @@ -25,7 +26,6 @@ in ${goldenDir} ${clientCidr}(rw,sync,no_subtree_check,fsid=0,no_root_squash) ''; - # NFSv4 only needs 2049/tcp (no rpcbind/mountd/statd), so the firewall stays - # minimal. The export itself is already restricted to the LAN CIDR above. - networking.firewall.allowedTCPPorts = [ 2049 ]; + # NFSv4 only needs 2049/tcp (no rpcbind/mountd/statd). The export is limited + # to the LAN, and the shared firewall trusts the LAN. } diff --git a/lib/genK3sAgentModule.nix b/lib/genK3sAgentModule.nix index a94b03da..af225501 100644 --- a/lib/genK3sAgentModule.nix +++ b/lib/genK3sAgentModule.nix @@ -25,6 +25,14 @@ in enableIPv6 = true; }; + # Cilium pods reach host services (e.g. the API server) from the pod CIDR. + # `mkAfter` keeps this after the shared base rules (incl. the node_exporter + # drop), so pods cannot scrape node_exporter either. + networking.firewall.extraInputRules = pkgs.lib.mkAfter '' + ip saddr 10.0.0.0/8 accept + ip6 saddr fd00::/104 accept + ''; + services.k3s = { enable = true; inherit package tokenFile; diff --git a/lib/genK3sServerModule.nix b/lib/genK3sServerModule.nix index f68f0712..3d1856e9 100644 --- a/lib/genK3sServerModule.nix +++ b/lib/genK3sServerModule.nix @@ -51,6 +51,15 @@ in enable = true; enableIPv6 = true; }; + + # Cilium pods reach host services (e.g. the API server) from the pod CIDR. + # `mkAfter` keeps this after the shared base rules (incl. the node_exporter + # drop), so pods cannot scrape node_exporter either. + networking.firewall.extraInputRules = lib.mkAfter '' + ip saddr 10.0.0.0/8 accept + ip6 saddr fd00::/104 accept + ''; + services.k3s = { enable = true; inherit package tokenFile clusterInit; diff --git a/modules/nixos/base/networking/firewall.nix b/modules/nixos/base/networking/firewall.nix index 0673fb60..879aceb2 100644 --- a/modules/nixos/base/networking/firewall.nix +++ b/modules/nixos/base/networking/firewall.nix @@ -1,20 +1,40 @@ +{ lib, myvars, ... }: +let + inherit (myvars.networking) lanCidr; + # The host that scrapes the exporters (VictoriaMetrics runs on youko). + monitoringHost = myvars.networking.hostsAddr.youko.ipv4; +in { - # for security reasons, only open the following ports to the network by default. - networking.firewall.allowedTCPPorts = [ - # localsend - 53317 + # Hosts bring their own firewall as defence in depth behind the router's + # (which stays on, with port mappings for the few ports we need): the home + # LAN and the tailnet are trusted equally (tighten the tailnet with Tailscale + # ACLs), and everything else is denied on both address families. + networking.nftables.enable = true; - # tcp ports for testing & sharing - 63080 - 63081 - 63082 - 63083 - 63084 - 63085 - 63086 - 63087 - 63088 - 63089 - ]; - # networking.firewall.allowedUDPPorts = [ ... ]; + networking.firewall = { + enable = true; + + # Nothing is opened to the Internet. LAN/tailnet/local-container access + # comes from the rules below; per-service `openFirewall` ports are + # suppressed. + allowedTCPPorts = lib.mkForce [ ]; + allowedUDPPorts = lib.mkForce [ ]; + + extraInputRules = '' + # node_exporter is scraped only by the monitoring host, not the whole + # LAN/tailnet/local containers. + ip saddr != ${monitoringHost} tcp dport 9100 drop + + # The local container bridge and the tailnet have the same access as the + # LAN. Kept after the drop above so neither can reach node_exporter. + iifname "podman0" accept + iifname "tailscale0" accept + + # The home LAN is the trusted core. + ip saddr ${lanCidr} accept + ip6 saddr fe80::/10 accept + # Tailscale's WireGuard port, for direct (non-DERP) connections. + udp dport 41641 accept + ''; + }; } diff --git a/modules/nixos/desktop/networking/firewall.nix b/modules/nixos/desktop/networking/firewall.nix deleted file mode 100644 index 601726f4..00000000 --- a/modules/nixos/desktop/networking/firewall.nix +++ /dev/null @@ -1,10 +0,0 @@ -# Desktops hold personal data and laptops join untrusted networks, so enable the -# firewall here (repo-wide default is off, see modules/nixos/base/ssh.nix). -# SSH/Tailscale/LocalSend/Sunshine keep working via their own `openFirewall` options. -{ - networking.firewall = { - enable = true; - # Trust everything from the tailnet. - trustedInterfaces = [ "tailscale0" ]; - }; -} diff --git a/modules/nixos/server/server-aarch64.nix b/modules/nixos/server/server-aarch64.nix index 7ac802cb..bbc9c0ae 100644 --- a/modules/nixos/server/server-aarch64.nix +++ b/modules/nixos/server/server-aarch64.nix @@ -15,16 +15,16 @@ ../base/ssh.nix ../base/user-group.nix + # This server imports base files individually instead of the whole + # modules/nixos/base group, so pull the shared firewall in explicitly. + ../base/networking/firewall.nix + ../../base # AppArmor is wired via modules/nixos/base/default.nix for other hosts; this # aarch64 server imports base files individually, so add it explicitly. (mylib.relativeToRoot "hardening/apparmor") ]; - # Servers run on the trusted internal LAN (NAT'd; WAN protected at the router). - # Keep the firewall off here; the secure default is ON (see modules/nixos/base/ssh.nix). - networking.firewall.enable = false; - # Fix: jasper is marked as broken, refusing to evaluate. environment.enableAllTerminfo = lib.mkForce false; } diff --git a/modules/nixos/server/server-riscv64.nix b/modules/nixos/server/server-riscv64.nix index eb433fab..cc117b77 100644 --- a/modules/nixos/server/server-riscv64.nix +++ b/modules/nixos/server/server-riscv64.nix @@ -13,6 +13,10 @@ ../base/ssh.nix ../base/user-group.nix + # This server imports base files individually instead of the whole + # modules/nixos/base group, so pull the shared firewall in explicitly. + ../base/networking/firewall.nix + ../../base ]; } diff --git a/modules/nixos/server/server.nix b/modules/nixos/server/server.nix index 3e95d887..c9434748 100644 --- a/modules/nixos/server/server.nix +++ b/modules/nixos/server/server.nix @@ -1,11 +1,11 @@ -{ lib, ... }: +{ ... }: { imports = [ ../base ../../base ]; - # Servers run on the trusted internal LAN (NAT'd; WAN protected at the router). - # Keep the firewall off here; the secure default is ON (see modules/nixos/base/ssh.nix). - networking.firewall.enable = false; + # Servers used to keep the firewall off, trusting the router alone. They now + # run the shared base firewall (modules/nixos/base/networking/firewall.nix) + # like every other host, as defence in depth behind the router. } diff --git a/outputs/aarch64-linux/tests/security-firewall/expected.nix b/outputs/aarch64-linux/tests/security-firewall/expected.nix index 6ca2c34e..81b3fbb5 100644 --- a/outputs/aarch64-linux/tests/security-firewall/expected.nix +++ b/outputs/aarch64-linux/tests/security-firewall/expected.nix @@ -2,6 +2,4 @@ lib, outputs, }: -lib.genAttrs (builtins.attrNames outputs.nixosConfigurations) ( - name: name == "ai-niri" || name == "shoukei-niri" -) +lib.genAttrs (builtins.attrNames outputs.nixosConfigurations) (_: true) diff --git a/outputs/aarch64-linux/tests/security-firewall/expr.nix b/outputs/aarch64-linux/tests/security-firewall/expr.nix index 39873f63..483dcb80 100644 --- a/outputs/aarch64-linux/tests/security-firewall/expr.nix +++ b/outputs/aarch64-linux/tests/security-firewall/expr.nix @@ -1,7 +1,18 @@ { + myvars, lib, outputs, }: lib.genAttrs (builtins.attrNames outputs.nixosConfigurations) ( - name: outputs.nixosConfigurations.${name}.config.networking.firewall.enable + name: + let + firewall = outputs.nixosConfigurations.${name}.config.networking.firewall; + in + # Every host must run the shared firewall (modules/nixos/base/networking/ + # firewall.nix): enabled, trusting the LAN by source, and opening nothing to + # the Internet through the global port lists. + firewall.enable + && lib.hasInfix myvars.networking.lanCidr firewall.extraInputRules + && firewall.allowedTCPPorts == [ ] + && firewall.allowedUDPPorts == [ ] ) diff --git a/outputs/x86_64-linux/tests/security-aquamarine-metrics/expected.nix b/outputs/x86_64-linux/tests/security-aquamarine-metrics/expected.nix index c45419fd..68785c7f 100644 --- a/outputs/x86_64-linux/tests/security-aquamarine-metrics/expected.nix +++ b/outputs/x86_64-linux/tests/security-aquamarine-metrics/expected.nix @@ -3,7 +3,6 @@ outputs, }: { - v2rayExporter = "127.0.0.1"; pgExporter = "127.0.0.1"; sftpgoTelemetry = "127.0.0.1"; } diff --git a/outputs/x86_64-linux/tests/security-aquamarine-metrics/expr.nix b/outputs/x86_64-linux/tests/security-aquamarine-metrics/expr.nix index ad81b217..65e0e966 100644 --- a/outputs/x86_64-linux/tests/security-aquamarine-metrics/expr.nix +++ b/outputs/x86_64-linux/tests/security-aquamarine-metrics/expr.nix @@ -6,7 +6,6 @@ let aqua = outputs.nixosConfigurations.youko.config; in { - v2rayExporter = aqua.services.prometheus.exporters.v2ray.listenAddress; pgExporter = aqua.services.prometheus.exporters.postgres.listenAddress; sftpgoTelemetry = aqua.services.sftpgo.settings.telemetry.bind_address; } diff --git a/outputs/x86_64-linux/tests/security-firewall/expected.nix b/outputs/x86_64-linux/tests/security-firewall/expected.nix index 6ca2c34e..81b3fbb5 100644 --- a/outputs/x86_64-linux/tests/security-firewall/expected.nix +++ b/outputs/x86_64-linux/tests/security-firewall/expected.nix @@ -2,6 +2,4 @@ lib, outputs, }: -lib.genAttrs (builtins.attrNames outputs.nixosConfigurations) ( - name: name == "ai-niri" || name == "shoukei-niri" -) +lib.genAttrs (builtins.attrNames outputs.nixosConfigurations) (_: true) diff --git a/outputs/x86_64-linux/tests/security-firewall/expr.nix b/outputs/x86_64-linux/tests/security-firewall/expr.nix index 39873f63..483dcb80 100644 --- a/outputs/x86_64-linux/tests/security-firewall/expr.nix +++ b/outputs/x86_64-linux/tests/security-firewall/expr.nix @@ -1,7 +1,18 @@ { + myvars, lib, outputs, }: lib.genAttrs (builtins.attrNames outputs.nixosConfigurations) ( - name: outputs.nixosConfigurations.${name}.config.networking.firewall.enable + name: + let + firewall = outputs.nixosConfigurations.${name}.config.networking.firewall; + in + # Every host must run the shared firewall (modules/nixos/base/networking/ + # firewall.nix): enabled, trusting the LAN by source, and opening nothing to + # the Internet through the global port lists. + firewall.enable + && lib.hasInfix myvars.networking.lanCidr firewall.extraInputRules + && firewall.allowedTCPPorts == [ ] + && firewall.allowedUDPPorts == [ ] ) diff --git a/vars/networking.nix b/vars/networking.nix index 3ef578ae..be4647e5 100644 --- a/vars/networking.nix +++ b/vars/networking.nix @@ -2,6 +2,8 @@ rec { mainGateway = "192.168.5.1"; # main router mainGateway6 = "fe80::5"; # main router's link-local address + # The trusted home LAN; the shared host firewall trusts this by source. + lanCidr = "192.168.5.0/24"; # use suzi as the default gateway # it's a subrouter with a transparent proxy proxyGateway = "192.168.5.178";