From 3acb5d4ee5c18a2c314228d4948fb59f69be3963 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?ryan4yin=20=7C=20=E4=BA=8C=E8=8A=B1?= Date: Wed, 7 Oct 2026 15:10:52 +0800 Subject: [PATCH] fix(nfs,libvirt): restrict the golden export and bind VNC to loopback (#400) * fix(nfs): restrict the golden export to its real clients * fix(libvirt): bind VNC consoles to loopback * fix(firewall): drop VNC ports on every host * docs(security): record the NFS and VNC boundaries --- SECURITY.md | 12 +++++--- WORKAROUNDS.md | 7 +++-- hosts/12kingdoms-youko/nfs-golden.nix | 34 +++++++++++++++------- lib/genLibvirtDomainXml.nix | 4 ++- modules/nixos/base/networking/firewall.nix | 5 ++++ tests/security-exporters.nix | 8 ++++- 6 files changed, 50 insertions(+), 20 deletions(-) diff --git a/SECURITY.md b/SECURITY.md index e63fabac..c3544620 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -194,10 +194,14 @@ checks remain necessary after deployment. privileges are intentional for cluster reconciliation; Git write access and controller credentials therefore remain administration boundaries. This inventory is not a complete effective-access or privilege-escalation analysis. -- **NFS:** the golden store is `0755 root:root`, exported read-write to the entire home LAN with - `sec=sys,no_root_squash`. File mode 0755 does not contain an allowed client with root privileges; - such a client can act as server root within this export. The network allowlist and client trust - are therefore material boundaries, not merely filesystem permissions. +- **NFS:** the golden store is `0755 root:root`, exported read-write to an explicit client list (the + k3s nodes and the VM hosts) with `sec=sys,no_root_squash`. The client list stops other LAN devices + reaching the export, but a root user on an allowed client still has server-root write access. + `root_squash` is deferred: the existing PVC subdirectories are root-owned (observed + `drwxr-xr-x root root` and `drwxrwsr-x root 107`), so enabling it needs a one-time ownership + migration and a maintenance window. See WORKAROUNDS.md. +- **Remote consoles:** the libvirt VNC consoles listen on `127.0.0.1` only, and the shared firewall + drops 5900-5910 on every host. Reach a console through an SSH tunnel. - **Pod-to-host firewall:** pod CIDR sources (`10.0.0.0/8`, `fd00::/104`) are accepted on the k3s nodes, but the credential-bearing host ports are denied first (SSH, NFS, Postgres, restic, VNC, Immich, and the exporter/monitoring ports). The broad accept stays so Cilium and in-cluster diff --git a/WORKAROUNDS.md b/WORKAROUNDS.md index d8720d26..2bc6d529 100644 --- a/WORKAROUNDS.md +++ b/WORKAROUNDS.md @@ -59,9 +59,10 @@ A known limitation with no fix yet. An unfinished feature or gap. -| ID | What & where | Why | Removal condition | Added | Revisit | Status | -| ------ | --------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------- | ---------- | --------- | ------ | -| WA-013 | NixOS tests — `outputs/README.md` | full-host tests still rebuild custom packages (no private cache) and need the host ssh key to decrypt agenix; the shared-firewall security-exporters VM test now runs in CI (KVM) | Resolve when a private cache exists and tests can run per-service | 2024-03-13 | quarterly | active | +| ID | What & where | Why | Removal condition | Added | Revisit | Status | +| ------ | --------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------- | ---------- | ----------------------- | ------ | +| WA-013 | NixOS tests — `outputs/README.md` | full-host tests still rebuild custom packages (no private cache) and need the host ssh key to decrypt agenix; the shared-firewall security-exporters VM test now runs in CI (KVM) | Resolve when a private cache exists and tests can run per-service | 2024-03-13 | quarterly | active | +| WA-020 | NFS golden store `no_root_squash` — `hosts/12kingdoms-youko/nfs-golden.nix` | the existing PVC subdirectories are root-owned, so enabling `root_squash` would break running workloads until they are migrated; the CSI node plugin is hostNetwork and sources from the allowlisted node IPs | Drop when the PVC subdirectories are migrated and `root_squash` is enabled | 2026-10-07 | next maintenance window | active | ## Related diff --git a/hosts/12kingdoms-youko/nfs-golden.nix b/hosts/12kingdoms-youko/nfs-golden.nix index 2ec4cc9a..e5ed2e81 100644 --- a/hosts/12kingdoms-youko/nfs-golden.nix +++ b/hosts/12kingdoms-youko/nfs-golden.nix @@ -1,19 +1,29 @@ { + myvars, + lib, ... }: let - # Shared golden-image store for the VM cluster. It is exported over - # NFSv4 so the cluster's NFS CSI driver can hand out RWX volumes to every - # node, and VM disks are cloned from it into local-path. + # Shared golden-image store for the VM cluster. It is exported over NFSv4 so + # the cluster's NFS CSI driver can hand out RWX volumes to every node, and VM + # disks are cloned from it into local-path. # # It lives on the encrypted btrfs pool (/persistent), which has plenty of # free space on this host. goldenDir = "/persistent/nfs/golden"; - # Only the homelab LAN may mount the export; the VM hosts and the k3s nodes - # (which run the NFS-CSI node plugin) are all on it. The store is not - # sensitive, so the whole LAN is fine rather than an explicit client list. - clientCidr = "192.168.5.0/24"; + # Only the hosts that actually mount the export may reach it: every k3s node + # runs the NFS CSI node plugin, and the VM hosts clone golden disks by hand. + # The rest of the LAN is not a client. + k3sNodes = lib.filterAttrs (name: _: lib.hasPrefix "k3s-test-1-" name) myvars.networking.hostsAddr; + clients = [ + myvars.networking.hostsAddr.shoryu.ipv4 + myvars.networking.hostsAddr.shushou.ipv4 + myvars.networking.hostsAddr.youko.ipv4 + ] + ++ lib.mapAttrsToList (_: v: v.ipv4) k3sNodes; + + exportOpts = "rw,sync,no_subtree_check,fsid=0,no_root_squash"; in { # Make sure the export directory exists before the NFS server starts. @@ -22,10 +32,12 @@ in ]; services.nfs.server.enable = true; - services.nfs.server.exports = '' - ${goldenDir} ${clientCidr}(rw,sync,no_subtree_check,fsid=0,no_root_squash) - ''; + services.nfs.server.exports = lib.concatStringsSep "\n" ( + map (client: "${goldenDir} ${client}(${exportOpts})") clients + ); # NFSv4 only needs 2049/tcp (no rpcbind/mountd/statd). The export is limited - # to the LAN, and the shared firewall trusts the LAN. + # to the hosts above, not the whole LAN. `no_root_squash` stays until the + # existing root-owned PVC subdirectories are migrated for `root_squash`; see + # SECURITY.md. } diff --git a/lib/genLibvirtDomainXml.nix b/lib/genLibvirtDomainXml.nix index bf28e416..d74c1f88 100644 --- a/lib/genLibvirtDomainXml.nix +++ b/lib/genLibvirtDomainXml.nix @@ -45,7 +45,9 @@
- + + + diff --git a/modules/nixos/base/networking/firewall.nix b/modules/nixos/base/networking/firewall.nix index 7b38bb6a..8966358f 100644 --- a/modules/nixos/base/networking/firewall.nix +++ b/modules/nixos/base/networking/firewall.nix @@ -38,6 +38,11 @@ in # this does not restrict other IPv6 services or loopback access. meta nfproto ipv6 tcp dport { 9100, 9835, 9633 } drop + # VNC consoles (libvirt on the VM hosts, and x11vnc inside the guests) + # are never exposed to the LAN, tailnet, or containers: reach them + # through an SSH tunnel. Loopback is accepted by the firewall's own rule. + tcp dport { 5900, 5901, 5902, 5903, 5910 } drop + # The local container bridge and the tailnet have the same access as the # LAN. Kept after the drop above so neither can reach node_exporter. iifname "podman0" accept diff --git a/tests/security-exporters.nix b/tests/security-exporters.nix index aac4689e..82ad5d3c 100644 --- a/tests/security-exporters.nix +++ b/tests/security-exporters.nix @@ -35,7 +35,7 @@ pkgs.testers.runNixOSTest { networking.firewall.extraInputRules = lib.mkAfter '' ip6 saddr fd42:1::/64 accept ''; - systemd.services = lib.genAttrs [ "9100" "9633" "9835" "8080" ] (port: { + systemd.services = lib.genAttrs [ "9100" "9633" "9835" "8080" "5902" ] (port: { wantedBy = [ "multi-user.target" ]; serviceConfig.ExecStart = "${pkgs.socat}/bin/socat TCP6-LISTEN:${port},ipv6only=0,reuseaddr,fork EXEC:${pkgs.coreutils}/bin/cat"; }); @@ -59,5 +59,11 @@ pkgs.testers.runNixOSTest { server.succeed(f"nc -6 -z -w 3 ::1 {port}") client.succeed("nc -z -w 3 192.168.5.200 8080") client.succeed("nc -6 -z -w 3 fd42:1::200 8080") + # VNC is never reachable off-host on either family; loopback still works. + monitor.fail("nc -z -w 3 192.168.5.200 5902") + client.fail("nc -z -w 3 192.168.5.200 5902") + monitor.fail("nc -6 -z -w 3 fd42:1::200 5902") + client.fail("nc -6 -z -w 3 fd42:1::200 5902") + server.succeed("nc -z -w 3 127.0.0.1 5902") ''; }