From f03907c6a47a5ce39332c14684e53beab7ab3b83 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?ryan4yin=20=7C=20=E4=BA=8C=E8=8A=B1?= Date: Sat, 3 Oct 2026 16:33:50 +0800 Subject: [PATCH] feat(security): add balanced kernel and exporter baseline (#365) --- SECURITY.md | 38 ++++---- modules/nixos/base/kernel-hardening.nix | 37 ++++++-- modules/nixos/base/networking/firewall.nix | 5 ++ modules/nixos/server/server-aarch64.nix | 1 + modules/nixos/server/server-riscv64.nix | 1 + .../tests/security-exporters/expected.nix | 2 + .../tests/security-exporters/expr.nix | 6 ++ .../tests/security-kernel/expected.nix | 2 + .../tests/security-kernel/expr.nix | 2 + outputs/default.nix | 87 +++++++++++-------- .../tests/security-exporters/expected.nix | 8 ++ .../tests/security-exporters/expr.nix | 58 +++++++++++++ .../tests/security-kernel/expected.nix | 18 ++++ .../tests/security-kernel/expr.nix | 30 +++++++ tests/security-exporters.nix | 63 ++++++++++++++ 15 files changed, 294 insertions(+), 64 deletions(-) create mode 100644 outputs/aarch64-linux/tests/security-exporters/expected.nix create mode 100644 outputs/aarch64-linux/tests/security-exporters/expr.nix create mode 100644 outputs/aarch64-linux/tests/security-kernel/expected.nix create mode 100644 outputs/aarch64-linux/tests/security-kernel/expr.nix create mode 100644 outputs/x86_64-linux/tests/security-exporters/expected.nix create mode 100644 outputs/x86_64-linux/tests/security-exporters/expr.nix create mode 100644 outputs/x86_64-linux/tests/security-kernel/expected.nix create mode 100644 outputs/x86_64-linux/tests/security-kernel/expr.nix create mode 100644 tests/security-exporters.nix diff --git a/SECURITY.md b/SECURITY.md index 52ff0f2c..20a65133 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -48,12 +48,12 @@ Sources: [host layout](hosts/README.md), [k3s deployment](hosts/k8s/README.md), [Nix trust](modules/base/nix.nix), [secrets](secrets/README.md), and [backup](BACKUP.md). Cluster resources are maintained in the separate k8s-gitops repository. -## Planned kernel and process baseline +## Kernel and process baseline -The following explicit, overridable baseline is planned for +The following explicit, overridable baseline is implemented in [kernel-hardening.nix](modules/nixos/base/kernel-hardening.nix), with Linux eval coverage and -explicit ARM/RISC-V server imports. This documentation does not enable those settings. Some are -already kernel/systemd/NixOS defaults; Darwin does not receive Linux sysctls. +explicit ARM/RISC-V server imports. Source implementation is not runtime verification. Some values +are already kernel/systemd/NixOS defaults; Darwin does not receive Linux sysctls. | Setting | Value | Effect and compatibility | | ------------------------------------------------- | ----- | ----------------------------------------------------------------------------------------------------------------------------------- | @@ -72,10 +72,10 @@ baseline reviewable; they do not imply the previous system had no protection. Un features may lack their sysctl paths, especially on custom kernels: runtime verification is required. -The planned JIT hardening is deliberately limited to unprivileged programs rather than forced on -privileged agents. Together with `unprivileged_bpf_disabled=2`, this preserves Cilium's normal -privileged BPF path while retaining protection if an administrator temporarily permits unprivileged -BPF. This is a configuration trade-off, not a measured claim of zero overhead. +JIT hardening is deliberately limited to unprivileged programs rather than forced on privileged +agents. Together with `unprivileged_bpf_disabled=2`, this preserves Cilium's normal privileged BPF +path while retaining protection if an administrator temporarily permits unprivileged BPF. This is a +configuration trade-off, not a measured claim of zero overhead. User namespaces remain enabled: disabling them globally conflicts with Nix sandboxing and breaks rootless containers and application sandboxes. Nonprivileged seccomp filters are not disabled by the @@ -156,22 +156,20 @@ checks remain necessary after deployment. | Component | Status | Runtime verification | | ---------------------------------------------------------------- | -------------------------------------------------- | --------------------------------------------------- | | Architecture, agent-first policy, audits and canonical API tests | Included in this documentation change | Only the dated observations below were checked live | -| Explicit sysctl baseline and IPv6 exporter defence | Planned in a separate baseline implementation | Not deployed or verified on running hosts | +| Explicit sysctl baseline and IPv6 exporter defence | Implemented in this baseline change | Not deployed or verified on running hosts | | Kernel-state reporter and textfile metrics | Planned in a separate observability implementation | Not deployed or verified on running hosts | -### Planned implementation checks +### Baseline implementation checks -- Add an explicit, overridable sysctl baseline, preserving user namespaces and privileged BPF - performance. -- Add IPv6 exporter denial before broad LAN/tailnet/container rules. The monitoring configuration - uses IPv4 targets for these ports, so no scrape endpoint migration is needed. -- Add x86_64/aarch64 eval checks for baseline values, normal host overrides and generated exporter +- An explicit, overridable sysctl baseline preserves user namespaces and privileged BPF performance. +- IPv6 exporter denial precedes broad LAN/tailnet/container rules. The monitoring configuration uses + IPv4 targets for these ports, so no scrape endpoint migration is needed. +- x86_64/aarch64 eval checks cover baseline values, normal host overrides and generated exporter rules. Canonical API endpoint checks for all six cluster nodes already accompany this document. -- Add a reusable isolated VM check exercising allowed monitoring IPv4 access, denied other IPv4/IPv6 +- A reusable isolated VM check exercises allowed monitoring IPv4 access, denied other IPv4/IPv6 access to all three exporter ports, retained IPv6 loopback and a reachable ordinary IPv4/IPv6 - service. The planned command is `nix build .#checks.x86_64-linux.security-exporters`, importing - only the shared firewall, not real-host secrets or workloads, and does not replace deployment-time - checks. + service. Run `nix build .#checks.x86_64-linux.security-exporters`, importing only the shared + firewall, not real-host secrets or workloads, and does not replace deployment-time checks. ### Read-only audit snapshot — 2026-10-03 @@ -306,7 +304,7 @@ fresh metrics. Guest profiles and host-side runner references require separate v | Priority | Work | Acceptance / constraints | | -------- | -------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| P1 | Implement, roll out and verify the planned baseline/exporter rules | Runtime sysctl readback, allowed IPv4 scrape and denied IPv6/non-monitoring scrape tests; preserve Nix/Podman/Cilium/NFS functionality | +| P1 | Roll out and verify the implemented baseline/exporter rules | Runtime sysctl readback, allowed IPv4 scrape and denied IPv6/non-monitoring scrape tests; preserve Nix/Podman/Cilium/NFS functionality | | P1 | Review router mappings and tailnet ACLs; narrow service exposure where warranted | Audit was limited to sampled host listeners; test intended and denied access without locking out deployment | | P1 | Review NFS client scope and root identity requirements | Restrict to required VM hosts/CSI nodes where practical; test provisioning, permissions and existing PVCs before changing squash semantics | | P1 | Narrow operator/secret access where justified; finish permission checks | Selected effective probes and ai/youko modes audited; validate broad secret-reader needs, remaining hosts, recipient scope and Darwin per-file ownership | diff --git a/modules/nixos/base/kernel-hardening.nix b/modules/nixos/base/kernel-hardening.nix index 034856de..269d4453 100644 --- a/modules/nixos/base/kernel-hardening.nix +++ b/modules/nixos/base/kernel-hardening.nix @@ -1,10 +1,35 @@ +{ lib, pkgs, ... }: { - config, - lib, - pkgs, - ... -}: -{ + # Keep this baseline independent of a particular kernel CVE. User namespaces + # remain available for Nix builds, rootless containers and application sandboxes. + # See SECURITY.md for compatibility constraints and post-deployment checks. + boot.kernel.sysctl = { + "kernel.dmesg_restrict" = lib.mkDefault 1; + # NixOS already sets mkDefault 1; use a stronger default, still overridable + # by an ordinary host definition, rather than conflicting at priority 1000. + "kernel.kptr_restrict" = lib.mkOverride 900 2; + # Allow debuggers to trace their own children; unrelated processes need + # CAP_SYS_PTRACE or an explicit PR_SET_PTRACER exception. + "kernel.yama.ptrace_scope" = lib.mkDefault 1; + # Upstream documents >=2 as blocking unprivileged kernel profiling. Do not + # assume distro-specific values 3/4 provide additional upstream protection. + "kernel.perf_event_paranoid" = lib.mkDefault 2; + # 2 disables unprivileged BPF but lets an administrator re-enable it without + # rebooting. Privileged Cilium/observability agents retain their BPF access. + "kernel.unprivileged_bpf_disabled" = lib.mkDefault 2; + # Limit JIT hardening to unprivileged programs: do not impose its overhead + # on privileged Cilium/observability workloads. Userns root is not host root. + "net.core.bpf_jit_harden" = lib.mkDefault 1; + "vm.unprivileged_userfaultfd" = lib.mkDefault 0; + + # Symlink/hardlink protection is boolean; FIFO/regular-file protection also + # supports 2, extending protection to group-writable sticky directories. + "fs.protected_symlinks" = lib.mkDefault 1; + "fs.protected_hardlinks" = lib.mkDefault 1; + "fs.protected_fifos" = lib.mkDefault 2; + "fs.protected_regular" = lib.mkDefault 2; + }; + # Blacklist unused kernel modules to shrink the attack surface. esp4/esp6/rxrpc # were the vectors for the Dirty Frag LPE (CVE-2026-43284 / CVE-2026-43500); # kept as defence in depth even though the upstream fix has long landed, since diff --git a/modules/nixos/base/networking/firewall.nix b/modules/nixos/base/networking/firewall.nix index fe405a1c..7b38bb6a 100644 --- a/modules/nixos/base/networking/firewall.nix +++ b/modules/nixos/base/networking/firewall.nix @@ -33,6 +33,11 @@ in # monitoring host. ip saddr != ${monitoringHost} tcp dport 9633 drop + # Monitoring uses the stable IPv4 address above. Deny IPv6 exporter + # connections before broad LAN/tailnet/container trust can accept them; + # this does not restrict other IPv6 services or loopback access. + meta nfproto ipv6 tcp dport { 9100, 9835, 9633 } drop + # The local container bridge and the tailnet have the same access as the # LAN. Kept after the drop above so neither can reach node_exporter. iifname "podman0" accept diff --git a/modules/nixos/server/server-aarch64.nix b/modules/nixos/server/server-aarch64.nix index f42d20f6..cbc1aaf1 100644 --- a/modules/nixos/server/server-aarch64.nix +++ b/modules/nixos/server/server-aarch64.nix @@ -8,6 +8,7 @@ ../base/btop.nix ../base/core.nix ../base/i18n.nix + ../base/kernel-hardening.nix ../base/monitoring.nix ../base/nix.nix ../base/packages.nix diff --git a/modules/nixos/server/server-riscv64.nix b/modules/nixos/server/server-riscv64.nix index 77cd5abf..2e40b4f4 100644 --- a/modules/nixos/server/server-riscv64.nix +++ b/modules/nixos/server/server-riscv64.nix @@ -8,6 +8,7 @@ ../base/btop.nix ../base/core.nix ../base/i18n.nix + ../base/kernel-hardening.nix ../base/monitoring.nix ../base/nix.nix ../base/packages.nix diff --git a/outputs/aarch64-linux/tests/security-exporters/expected.nix b/outputs/aarch64-linux/tests/security-exporters/expected.nix new file mode 100644 index 00000000..252f05b1 --- /dev/null +++ b/outputs/aarch64-linux/tests/security-exporters/expected.nix @@ -0,0 +1,2 @@ +{ lib, outputs }: +import ../../../x86_64-linux/tests/security-exporters/expected.nix { inherit lib outputs; } diff --git a/outputs/aarch64-linux/tests/security-exporters/expr.nix b/outputs/aarch64-linux/tests/security-exporters/expr.nix new file mode 100644 index 00000000..320521e4 --- /dev/null +++ b/outputs/aarch64-linux/tests/security-exporters/expr.nix @@ -0,0 +1,6 @@ +{ + lib, + myvars, + outputs, +}: +import ../../../x86_64-linux/tests/security-exporters/expr.nix { inherit lib myvars outputs; } diff --git a/outputs/aarch64-linux/tests/security-kernel/expected.nix b/outputs/aarch64-linux/tests/security-kernel/expected.nix new file mode 100644 index 00000000..39f60583 --- /dev/null +++ b/outputs/aarch64-linux/tests/security-kernel/expected.nix @@ -0,0 +1,2 @@ +{ lib, outputs }: +import ../../../x86_64-linux/tests/security-kernel/expected.nix { inherit lib outputs; } diff --git a/outputs/aarch64-linux/tests/security-kernel/expr.nix b/outputs/aarch64-linux/tests/security-kernel/expr.nix new file mode 100644 index 00000000..289e3ed3 --- /dev/null +++ b/outputs/aarch64-linux/tests/security-kernel/expr.nix @@ -0,0 +1,2 @@ +{ lib, outputs }: +import ../../../x86_64-linux/tests/security-kernel/expr.nix { inherit lib outputs; } diff --git a/outputs/default.nix b/outputs/default.nix index 2d77f101..63c316c3 100644 --- a/outputs/default.nix +++ b/outputs/default.nix @@ -152,49 +152,60 @@ in # Eval Tests for all NixOS & darwin systems. evalTests = lib.lists.all (it: it.evalTests == { }) allSystemValues; - checks = forAllSystems (system: { - # eval-tests per system. `nix flake check` requires every check to be a - # derivation, so wrap the boolean result in one instead of returning a bool. - eval-tests = - let - pkgs = nixpkgs.legacyPackages.${system}; - results = allSystems.${system}.evalTests; - in - pkgs.runCommand "eval-tests" { } ( - if results == { } then - "touch $out" - else - "echo 'eval tests failed: evalTests is not empty' >&2; exit 1" - ); + checks = forAllSystems ( + system: + { + # eval-tests per system. `nix flake check` requires every check to be a + # derivation, so wrap the boolean result in one instead of returning a bool. + eval-tests = + let + pkgs = nixpkgs.legacyPackages.${system}; + results = allSystems.${system}.evalTests; + in + pkgs.runCommand "eval-tests" { } ( + if results == { } then + "touch $out" + else + "echo 'eval tests failed: evalTests is not empty' >&2; exit 1" + ); - pre-commit-check = pre-commit-hooks.lib.${system}.run { - src = mylib.relativeToRoot "."; - hooks = { - nixfmt = { - enable = true; - settings.width = 100; - }; - # Source code spell checker - typos = { - enable = true; - settings = { - write = true; # Automatically fix typos - configPath = ".typos.toml"; # relative to the flake root - exclude = "rime-data/"; + pre-commit-check = pre-commit-hooks.lib.${system}.run { + src = mylib.relativeToRoot "."; + hooks = { + nixfmt = { + enable = true; + settings.width = 100; }; - }; - prettier = { - enable = true; - settings = { - write = true; # Automatically format files - configPath = ".prettierrc.yaml"; # relative to the flake root + # Source code spell checker + typos = { + enable = true; + settings = { + write = true; # Automatically fix typos + configPath = ".typos.toml"; # relative to the flake root + exclude = "rime-data/"; + }; }; + prettier = { + enable = true; + settings = { + write = true; # Automatically format files + configPath = ".prettierrc.yaml"; # relative to the flake root + }; + }; + # deadnix.enable = true; # detect unused variable bindings in `*.nix` + # statix.enable = true; # lints and suggestions for Nix code(auto suggestions) }; - # deadnix.enable = true; # detect unused variable bindings in `*.nix` - # statix.enable = true; # lints and suggestions for Nix code(auto suggestions) }; - }; - }); + } + // lib.optionalAttrs (system == "x86_64-linux") { + # Test the shared firewall in disposable VMs, without real-host imports, + # Home Manager or agenix identities. Linux eval tests remain cross-arch. + security-exporters = import ../tests/security-exporters.nix { + pkgs = nixpkgs.legacyPackages.${system}; + networking = myvars.networking; + }; + } + ); # Development Shells devShells = forAllSystems ( diff --git a/outputs/x86_64-linux/tests/security-exporters/expected.nix b/outputs/x86_64-linux/tests/security-exporters/expected.nix new file mode 100644 index 00000000..7475ecd0 --- /dev/null +++ b/outputs/x86_64-linux/tests/security-exporters/expected.nix @@ -0,0 +1,8 @@ +{ lib, outputs }: +lib.genAttrs (builtins.attrNames outputs.nixosConfigurations) (_: { + ipv4Restricted = true; + ipv6RestrictedBeforeTrust = true; + generatedRestrictionsBeforeTrust = true; + lanRetained = true; + tailnetRetained = true; +}) diff --git a/outputs/x86_64-linux/tests/security-exporters/expr.nix b/outputs/x86_64-linux/tests/security-exporters/expr.nix new file mode 100644 index 00000000..be804dcb --- /dev/null +++ b/outputs/x86_64-linux/tests/security-exporters/expr.nix @@ -0,0 +1,58 @@ +{ + lib, + myvars, + outputs, +}: +lib.genAttrs (builtins.attrNames outputs.nixosConfigurations) ( + name: + let + config = outputs.nixosConfigurations.${name}.config; + rules = config.networking.firewall.extraInputRules; + # Evaluate the actual merged rule ordering, including k3s pod-network rules. + beforeTrust = builtins.head (lib.splitString "iifname \"podman0\" accept" rules); + ipv6Guard = "meta nfproto ipv6 tcp dport { 9100, 9835, 9633 } drop"; + monitoringHost = myvars.networking.hostsAddr.youko.ipv4; + generated = config.networking.nftables.tables.nixos-fw.content; + # rpfilter accepts valid source routes before input filtering; that is not + # service access. Compare trust/denial ordering within input-allow only. + inputAllow = builtins.head ( + lib.splitString "\n}" (lib.last (lib.splitString "chain input-allow {" generated)) + ); + broadTrustRules = lib.filter ( + line: + lib.hasSuffix " accept" line + && ( + lib.hasInfix "saddr" line || lib.hasInfix "\"podman0\"" line || lib.hasInfix "\"tailscale0\"" line + ) + ) (map lib.strings.trim (lib.splitString "\n" inputAllow)); + exporterGuards = [ + ipv6Guard + ] + ++ map (port: "ip saddr != ${monitoringHost} tcp dport ${toString port} drop") [ + 9100 + 9835 + 9633 + ]; + in + { + ipv4Restricted = + lib.all + (port: lib.hasInfix "ip saddr != ${monitoringHost} tcp dport ${toString port} drop" beforeTrust) + [ + 9100 + 9835 + 9633 + ]; + ipv6RestrictedBeforeTrust = lib.hasInfix ipv6Guard beforeTrust; + generatedRestrictionsBeforeTrust = + broadTrustRules != [ ] + && lib.all ( + trust: + lib.all ( + guard: lib.hasInfix guard (builtins.head (lib.splitString trust inputAllow)) + ) exporterGuards + ) broadTrustRules; + lanRetained = lib.hasInfix "ip saddr ${myvars.networking.lanCidr} accept" rules; + tailnetRetained = lib.hasInfix "iifname \"tailscale0\" accept" rules; + } +) diff --git a/outputs/x86_64-linux/tests/security-kernel/expected.nix b/outputs/x86_64-linux/tests/security-kernel/expected.nix new file mode 100644 index 00000000..0829dd70 --- /dev/null +++ b/outputs/x86_64-linux/tests/security-kernel/expected.nix @@ -0,0 +1,18 @@ +{ lib, outputs }: +lib.genAttrs (builtins.attrNames outputs.nixosConfigurations) (_: { + sysctl = { + "kernel.dmesg_restrict" = 1; + "kernel.kptr_restrict" = 2; + "kernel.yama.ptrace_scope" = 1; + "kernel.perf_event_paranoid" = 2; + "kernel.unprivileged_bpf_disabled" = 2; + "net.core.bpf_jit_harden" = 1; + "vm.unprivileged_userfaultfd" = 0; + "fs.protected_symlinks" = 1; + "fs.protected_hardlinks" = 1; + "fs.protected_fifos" = 2; + "fs.protected_regular" = 2; + }; + allowUserNamespaces = true; + hostOverride = 1; +}) diff --git a/outputs/x86_64-linux/tests/security-kernel/expr.nix b/outputs/x86_64-linux/tests/security-kernel/expr.nix new file mode 100644 index 00000000..28a14418 --- /dev/null +++ b/outputs/x86_64-linux/tests/security-kernel/expr.nix @@ -0,0 +1,30 @@ +{ lib, outputs }: +lib.genAttrs (builtins.attrNames outputs.nixosConfigurations) ( + name: + let + config = outputs.nixosConfigurations.${name}.config; + keys = [ + "kernel.dmesg_restrict" + "kernel.kptr_restrict" + "kernel.yama.ptrace_scope" + "kernel.perf_event_paranoid" + "kernel.unprivileged_bpf_disabled" + "net.core.bpf_jit_harden" + "vm.unprivileged_userfaultfd" + "fs.protected_symlinks" + "fs.protected_hardlinks" + "fs.protected_fifos" + "fs.protected_regular" + ]; + in + { + sysctl = lib.genAttrs keys (key: config.boot.kernel.sysctl.${key} or null); + # Nix sandbox, rootless Podman and application sandboxes need user namespaces. + inherit (config.security) allowUserNamespaces; + # An ordinary host exception must win without mkForce or a merge conflict. + hostOverride = + (outputs.nixosConfigurations.${name}.extendModules { + modules = [ { boot.kernel.sysctl."kernel.kptr_restrict" = 1; } ]; + }).config.boot.kernel.sysctl."kernel.kptr_restrict"; + } +) diff --git a/tests/security-exporters.nix b/tests/security-exporters.nix new file mode 100644 index 00000000..aac4689e --- /dev/null +++ b/tests/security-exporters.nix @@ -0,0 +1,63 @@ +{ pkgs, networking }: +let + inherit (pkgs) lib; + myvars = { inherit networking; }; + node = ipv4: ipv6: { + networking.interfaces.eth1 = { + ipv4.addresses = lib.mkForce [ + { + address = ipv4; + prefixLength = 24; + } + ]; + ipv6.addresses = [ + { + address = ipv6; + prefixLength = 64; + } + ]; + }; + environment.systemPackages = [ pkgs.netcat-openbsd ]; + virtualisation.memorySize = 384; + }; +in +pkgs.testers.runNixOSTest { + name = "security-exporter-dual-stack"; + nodes = { + server = { + imports = [ + ../modules/nixos/base/networking/firewall.nix + (node "192.168.5.200" "fd42:1::200") + ]; + _module.args = { inherit myvars; }; + # Model trusted IPv6 traffic after the exporter guards, like the tailnet + # or k3s pod-network trust. Other IPv6 services must remain usable. + networking.firewall.extraInputRules = lib.mkAfter '' + ip6 saddr fd42:1::/64 accept + ''; + systemd.services = lib.genAttrs [ "9100" "9633" "9835" "8080" ] (port: { + wantedBy = [ "multi-user.target" ]; + serviceConfig.ExecStart = "${pkgs.socat}/bin/socat TCP6-LISTEN:${port},ipv6only=0,reuseaddr,fork EXEC:${pkgs.coreutils}/bin/cat"; + }); + }; + monitor = node networking.hostsAddr.youko.ipv4 "fd42:1::183"; + client = node "192.168.5.199" "fd42:1::199"; + }; + testScript = '' + start_all() + server.wait_for_unit("multi-user.target") + monitor.wait_for_unit("multi-user.target") + client.wait_for_unit("multi-user.target") + for port in (9100, 9633, 9835, 8080): + server.wait_for_unit(f"{port}.service") + server.wait_for_open_port(port) + for port in (9100, 9633, 9835): + monitor.succeed(f"nc -z -w 3 192.168.5.200 {port}") + client.fail(f"nc -z -w 3 192.168.5.200 {port}") + monitor.fail(f"nc -6 -z -w 3 fd42:1::200 {port}") + client.fail(f"nc -6 -z -w 3 fd42:1::200 {port}") + server.succeed(f"nc -6 -z -w 3 ::1 {port}") + client.succeed("nc -z -w 3 192.168.5.200 8080") + client.succeed("nc -6 -z -w 3 fd42:1::200 8080") + ''; +}