From 7dc4e7a39f9f5a0ac0139b7e0e106b0d6063e857 Mon Sep 17 00:00:00 2001 From: Ryan Yin Date: Sat, 31 Oct 2026 01:09:15 +0800 Subject: [PATCH] feat(security): harden homelab SSH, services and audit trail The homelab hosts accept root SSH with long-lived file keys, run several exposed services with almost no systemd sandboxing, and keep no audit trail of privileged actions. Close those three gaps, weighing each measure against its cost instead of adopting a policy wholesale. SSH - Turn off KbdInteractiveAuthentication: nixpkgs defaults it to true and PAM then still accepts the Unix password despite PasswordAuthentication = false. - authorizedKeysInHomedir = false: only keys declared in Nix take effect; youko keeps Gitea's forced-command file through `Match User gitea`. - Pin aquamarine's passphrase-free remote-builder keys to `restrict,command="nix-daemon --stdio --store daemon"`, so builds run in the system daemon instead of an audited root login session. systemd sandboxing - Add the mylib.systemdHardening preset and apply it to the exposed, weakly sandboxed services: caddy 7.8 -> 1.5, vmalert 8.2 -> 1.4, rustfs 6.6 -> 1.4, whisper-asr 7.7 -> 1.6. An eval test pins the key restrictions. - Rejected: a global all-keys policy (<=0.2 score gains on services nixpkgs already sandboxes) and per-service confinement (bind paths to maintain). auditd on all four hosts, reusing nixpkgs' loader and unit - Audit login sessions only: a first `-a never,task -F auid=unset` rule keeps services out of the per-syscall audit path. A compromised service is therefore unrecorded until it opens a session. - Record root commands, credential and host-key access, writes to SSH, account, PAM, Nix-trust and audit configuration, module loads and session denials. - Rejected: global denial logging, service-UID watches and --loginuid-immutable (the kernel already records loginuid changes). Verified on all four hosts after deployment: zero failed units, k3s 6/6 Ready, and the exposed services healthy. Assisted-by: Claude Code (claude-opus-5-5) --- SECURITY.md | 54 ++++++-- hosts/12kingdoms-shoryu/default.nix | 1 + hosts/12kingdoms-shushou/default.nix | 1 + hosts/12kingdoms-youko/default.nix | 1 + .../homelab-services/caddy.nix | 15 +- .../homelab-services/gitea.nix | 7 + .../homelab-services/monitoring/alert.nix | 5 +- .../homelab-services/rustfs.nix | 20 ++- hosts/idols-aquamarine/asr.nix | 18 ++- hosts/idols-aquamarine/default.nix | 8 +- lib/README.md | 8 +- lib/default.nix | 2 + lib/systemdHardening.nix | 47 +++++++ modules/README.md | 1 + modules/nixos/base/auditd.nix | 128 ++++++++++++++++++ modules/nixos/base/ssh.nix | 4 + .../tests/security-audit/expected.nix | 2 + .../tests/security-audit/expr.nix | 2 + .../tests/security-ssh-auth/expected.nix | 25 ++++ .../tests/security-ssh-auth/expr.nix | 21 +++ outputs/default.nix | 6 + .../tests/security-audit/expected.nix | 24 ++++ .../tests/security-audit/expr.nix | 16 +++ .../security-service-hardening/expected.nix | 24 ++++ .../tests/security-service-hardening/expr.nix | 21 +++ .../tests/security-ssh-auth/expected.nix | 25 ++++ .../tests/security-ssh-auth/expr.nix | 21 +++ tests/security-auditd.nix | 100 ++++++++++++++ 28 files changed, 578 insertions(+), 29 deletions(-) create mode 100644 lib/systemdHardening.nix create mode 100644 modules/nixos/base/auditd.nix create mode 100644 outputs/aarch64-linux/tests/security-audit/expected.nix create mode 100644 outputs/aarch64-linux/tests/security-audit/expr.nix create mode 100644 outputs/aarch64-linux/tests/security-ssh-auth/expected.nix create mode 100644 outputs/aarch64-linux/tests/security-ssh-auth/expr.nix create mode 100644 outputs/x86_64-linux/tests/security-audit/expected.nix create mode 100644 outputs/x86_64-linux/tests/security-audit/expr.nix create mode 100644 outputs/x86_64-linux/tests/security-service-hardening/expected.nix create mode 100644 outputs/x86_64-linux/tests/security-service-hardening/expr.nix create mode 100644 outputs/x86_64-linux/tests/security-ssh-auth/expected.nix create mode 100644 outputs/x86_64-linux/tests/security-ssh-auth/expr.nix create mode 100644 tests/security-auditd.nix diff --git a/SECURITY.md b/SECURITY.md index e61f5bb7..75e29648 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -33,18 +33,18 @@ small, reversible controls and narrow exceptions; benchmark before adopting expe ## Architecture and trust boundaries -| Boundary | Current design | Limitation | -| --------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| Administration | Key-based SSH; root key login retained for Colmena/MicroVM deployment | Deployment credentials are root-equivalent; password login being disabled does not restrict stolen keys | -| Host network | Shared nftables firewall trusts the home LAN, tailnet interface and local Podman bridge; IPv6 link-local is accepted | These are broad trust zones, not service-level least privilege; router exposure and tailnet ACLs require separate verification | -| Monitoring | node/smartctl exporters bind the host LAN IPv4 (9100/9633); firewall rules restrict exporter ports 9100, 9835 and 9633 to the monitoring host over IPv4 | nvidia_gpu_exporter (9835) still binds wildcard; the IPv6 denial for those three ports is in the shared firewall; current running hosts require separate verification | -| VM hosts | shoryu, shushou and youko run guests behind `br0`; ruby/kana are libvirt guests | A bridge is not network isolation; host compromise affects its guests | -| k3s | Three control-plane and three worker MicroVMs; Cilium, API VIP and NFS CSI | Cluster-admin, privileged/host-mounted pods and writable NFS data are powerful trust boundaries | -| MicroVM storage | Host store shared read-only through virtiofs; guest `/etc`, `/var` and `/home` persisted in separate images | Read-only store does not isolate the host-side virtiofs implementation or protect writable guest state | -| Desktop apps | Selected apps use nixpak/bubblewrap; Podman is rootless | Rules and shared directories vary per app; user namespaces are needed for these sandboxes | -| Secrets | agenix declarations here, encrypted data in the private nix-secrets repository | Recipients and plaintext file permissions must be reviewed; root on a recipient can read its decrypted secrets | -| Supply chain | Locked flake inputs; explicit binary-cache trust keys on NixOS; normal user is not a Nix trusted-user | A trusted cache or imported Nix module can supply code executed as root; a lock file is not a security audit | -| Recovery | NixOS generations, btrbk/restic, hourly read-only snapshots of the REST store, and documented backup procedures | A rollback does not undo compromised data or secrets; backup configuration alone does not prove restore works; the offsite immutable copy is still planned | +| Boundary | Current design | Limitation | +| --------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Administration | Key-based SSH; root key login retained for Colmena/MicroVM deployment; only keys declared in Nix are honoured, and no password or keyboard-interactive login; remote-builder host keys only reach `nix-daemon` | Deployment credentials are root-equivalent file keys, usable by any same-user process through the desktop SSH agent; one stolen key reaches every host | +| Host network | Shared nftables firewall trusts the home LAN, tailnet interface and local Podman bridge; IPv6 link-local is accepted | These are broad trust zones, not service-level least privilege; router exposure and tailnet ACLs require separate verification | +| Monitoring | node/smartctl exporters bind the host LAN IPv4 (9100/9633); firewall rules restrict exporter ports 9100, 9835 and 9633 to the monitoring host over IPv4 | nvidia_gpu_exporter (9835) still binds wildcard; the IPv6 denial for those three ports is in the shared firewall; current running hosts require separate verification | +| VM hosts | shoryu, shushou and youko run guests behind `br0`; ruby/kana are libvirt guests | A bridge is not network isolation; host compromise affects its guests | +| k3s | Three control-plane and three worker MicroVMs; Cilium, API VIP and NFS CSI | Cluster-admin, privileged/host-mounted pods and writable NFS data are powerful trust boundaries | +| MicroVM storage | Host store shared read-only through virtiofs; guest `/etc`, `/var` and `/home` persisted in separate images | Read-only store does not isolate the host-side virtiofs implementation or protect writable guest state | +| Desktop apps | Selected apps use nixpak/bubblewrap; Podman is rootless | Rules and shared directories vary per app; user namespaces are needed for these sandboxes | +| Secrets | agenix declarations here, encrypted data in the private nix-secrets repository | Recipients and plaintext file permissions must be reviewed; root on a recipient can read its decrypted secrets | +| Supply chain | Locked flake inputs; explicit binary-cache trust keys on NixOS; normal user is not a Nix trusted-user | A trusted cache or imported Nix module can supply code executed as root; a lock file is not a security audit | +| Recovery | NixOS generations, btrbk/restic, hourly read-only snapshots of the REST store, and documented backup procedures | A rollback does not undo compromised data or secrets; backup configuration alone does not prove restore works; the offsite immutable copy is still planned | Sources: [host layout](hosts/README.md), [k3s deployment](hosts/k8s/README.md), [firewall](modules/nixos/base/networking/firewall.nix), [SSH](modules/nixos/base/ssh.nix), @@ -104,11 +104,35 @@ an advisory and affected kernel versions before adding CVE claims; search snippe - AppArmor is enabled, but the only explicit policy, sudo's, is in complain mode. Complain mode logs rather than blocks. Existing FHS-oriented abstractions need Nix store path review; this is not broad enforced application confinement. -- Systemd service sandboxing is per-service, not a global policy. Review capabilities, writable - paths, network access and syscall requirements before tightening an individual service. +- Systemd service sandboxing is per-service, not a global policy. The exposed services nixpkgs + leaves weakly sandboxed (caddy, rustfs, vmalert, whisper-asr) use the preset in + [systemdHardening.nix](lib/systemdHardening.nix). Review capabilities, writable paths, network + access and syscall requirements before tightening an individual service. See [application hardening](hardening/README.md) for wrapper details. +### Audit trail + +`modules.security.audit` ([auditd.nix](modules/nixos/base/auditd.nix)) runs auditd on youko, shoryu, +shushou and aquamarine. It records, for login sessions only (SSH, and sudo or su from one): every +root command (`root-session`), access to decrypted secrets and SSH host keys (`credential-access`), +writes to SSH, account/PAM, Nix trust and audit configuration, module loads, and denials. + +- Services are not audited: a `never,task` rule gives tasks without a login uid no audit context, so + virtiofsd, hypervisors, NFS, PostgreSQL and Nix builds skip the per-syscall audit path. A + compromised service is therefore invisible until it opens a login session. +- Logs stay on the host and root can erase them; shipping them off-host is a P1 TODO. +- One watch on a missing path makes `auditctl -R` stop and leaves auditing off; + `nix build .#checks.x86_64-linux.security-auditd` loads the rules in a VM. + +Query as root; sshd is built without libaudit, so use PAM's `USER_START` records for logins: + +```sh +ausearch -i -m USER_START --start today # logins with address and session id +ausearch -i -k root-session --start today # root commands, with auid and ses +ausearch -i --session # everything one session did +``` + ## Updating, deploying and recovering 1. Check the advisory and affected package/kernel; prioritize active exploitation and reachable @@ -355,6 +379,8 @@ separate verification. | Priority | Work | Acceptance / constraints | | -------- | -------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| P1 | Move root/deploy SSH to hardware-backed keys | `sk-ssh-ed25519` keys with touch for root, with SSH multiplexing to keep Colmena usable; an agent lifetime or confirmation; a tested break-glass path | +| P1 | Alert on SSH logins and ship auth/audit logs off-host | Off-host copy host root cannot rewrite; alert on an unknown key or source, a session-rate spike and `root-session`/`credential-access`; redact key material | | P1 | Roll out and verify the implemented baseline/exporter rules | Runtime sysctl readback, allowed IPv4 scrape and denied IPv6/non-monitoring scrape tests; preserve Nix/Podman/Cilium/NFS functionality | | P1 | Review router mappings and tailnet ACLs; narrow service exposure where warranted | Audit was limited to sampled host listeners; test intended and denied access without locking out deployment | | P1 | Review NFS client scope and root identity requirements | Restrict to required VM hosts/CSI nodes where practical; test provisioning, permissions and existing PVCs before changing squash semantics | diff --git a/hosts/12kingdoms-shoryu/default.nix b/hosts/12kingdoms-shoryu/default.nix index dc513c8d..f300a827 100644 --- a/hosts/12kingdoms-shoryu/default.nix +++ b/hosts/12kingdoms-shoryu/default.nix @@ -25,6 +25,7 @@ in ]; modules.btrbk.enable = true; + modules.security.audit.enable = true; # MoreFine S500Plus: Radeon iGPU. The ROCm build of btop shows the GPU panel; # node_exporter's drm collector exposes its utilization/VRAM. diff --git a/hosts/12kingdoms-shushou/default.nix b/hosts/12kingdoms-shushou/default.nix index 1957e286..d04f10db 100644 --- a/hosts/12kingdoms-shushou/default.nix +++ b/hosts/12kingdoms-shushou/default.nix @@ -24,6 +24,7 @@ in ]; modules.btrbk.enable = true; + modules.security.audit.enable = true; # MinisForum UM560: Radeon iGPU. The ROCm build of btop shows the GPU panel; # node_exporter's drm collector exposes its utilization/VRAM. diff --git a/hosts/12kingdoms-youko/default.nix b/hosts/12kingdoms-youko/default.nix index f33ec3c0..c3095f12 100644 --- a/hosts/12kingdoms-youko/default.nix +++ b/hosts/12kingdoms-youko/default.nix @@ -58,6 +58,7 @@ in }; modules.btrbk.enable = true; + modules.security.audit.enable = true; # HDD tuning/diagnostics for the two spinning disks behind the USB-SATA bridge. environment.systemPackages = [ pkgs.hdparm ]; diff --git a/hosts/12kingdoms-youko/homelab-services/caddy.nix b/hosts/12kingdoms-youko/homelab-services/caddy.nix index 02a08cd8..b56f2413 100644 --- a/hosts/12kingdoms-youko/homelab-services/caddy.nix +++ b/hosts/12kingdoms-youko/homelab-services/caddy.nix @@ -232,11 +232,22 @@ in }; }; - # caddy loads the lego-issued files at startup, so wait for the acme service; - # otherwise its config fails to load before the cert exists. systemd.services.caddy = { + # caddy loads the lego-issued files at startup, so wait for the acme + # service; otherwise its config fails to load before the cert exists. wants = [ "acme-writefor.fun.service" ]; after = [ "acme-writefor.fun.service" ]; + + # Sandbox the LAN front door; it writes only its State/LogsDirectory. + serviceConfig = mylib.systemdHardening.mkDefaults mylib.systemdHardening.service // { + # Only 80/443. No CAP_NET_ADMIN: the module already raises the UDP buffer + # sysctls quic-go would force. "" resets the package unit's list. + AmbientCapabilities = [ + "" + "CAP_NET_BIND_SERVICE" + ]; + CapabilityBoundingSet = [ "CAP_NET_BIND_SERVICE" ]; + }; }; # caddy is reached from the LAN/tailnet only; the shared firewall diff --git a/hosts/12kingdoms-youko/homelab-services/gitea.nix b/hosts/12kingdoms-youko/homelab-services/gitea.nix index 10f022f7..b0e6aaef 100644 --- a/hosts/12kingdoms-youko/homelab-services/gitea.nix +++ b/hosts/12kingdoms-youko/homelab-services/gitea.nix @@ -1,5 +1,12 @@ { pkgs, ... }: { + # Git over SSH reads Gitea's own ~/.ssh/authorized_keys (keys pinned to + # `gitea serv`), which modules/nixos/base/ssh.nix otherwise ignores. + services.openssh.extraConfig = '' + Match User gitea + AuthorizedKeysFile %h/.ssh/authorized_keys /etc/ssh/authorized_keys.d/%u + ''; + # https://github.com/NixOS/nixpkgs/blob/nixos-26.05/nixos/modules/services/misc/gitea.nix services.gitea = { enable = true; diff --git a/hosts/12kingdoms-youko/homelab-services/monitoring/alert.nix b/hosts/12kingdoms-youko/homelab-services/monitoring/alert.nix index 912ed747..3f092fcb 100644 --- a/hosts/12kingdoms-youko/homelab-services/monitoring/alert.nix +++ b/hosts/12kingdoms-youko/homelab-services/monitoring/alert.nix @@ -1,4 +1,4 @@ -{ config, ... }: +{ config, mylib, ... }: { # https://docs.victoriametrics.com/victoriametrics/vmalert/ services.vmalert.instances."homelab" = { @@ -34,6 +34,9 @@ }; }; + # nixpkgs only sets DynamicUser; vmalert talks to loopback and keeps no state. + systemd.services.vmalert-homelab.serviceConfig = mylib.systemdHardening.mkDefaults mylib.systemdHardening.service; + services.prometheus.alertmanager = { enable = true; listenAddress = "127.0.0.1"; diff --git a/hosts/12kingdoms-youko/homelab-services/rustfs.nix b/hosts/12kingdoms-youko/homelab-services/rustfs.nix index c137e1ee..a637cc0e 100644 --- a/hosts/12kingdoms-youko/homelab-services/rustfs.nix +++ b/hosts/12kingdoms-youko/homelab-services/rustfs.nix @@ -1,7 +1,11 @@ { config, + mylib, ... }: +let + inherit (mylib.systemdHardening) mkDefaults service; +in { # The service module comes from the default nixpkgs (services.rustfs); it # creates the rustfs user/group, the data volume via tmpfiles, and a hardened @@ -26,6 +30,18 @@ environmentFile = config.age.secrets."rustfs.env".path; }; - # The data volume lives on the encrypted HDD subvolume; wait for its mount. - systemd.services.rustfs.unitConfig.RequiresMountsFor = "/data/apps"; + systemd.services.rustfs = { + # The data volume lives on the encrypted HDD subvolume; wait for its mount. + unitConfig.RequiresMountsFor = "/data/apps"; + + # Fill the sandboxing the nixpkgs unit leaves out. + serviceConfig = mkDefaults service // { + # ProtectSystem=strict: the data volume is the only writable path. + ReadWritePaths = [ config.services.rustfs.settings.RUSTFS_VOLUMES ]; + # Its startup endpoint list enumerates the host's interface addresses. + RestrictAddressFamilies = service.RestrictAddressFamilies ++ [ "AF_NETLINK" ]; + # Capacity and observability metrics read host-wide /proc files. + ProcSubset = "all"; + }; + }; } diff --git a/hosts/idols-aquamarine/asr.nix b/hosts/idols-aquamarine/asr.nix index 3c2d9e3a..99d9e393 100644 --- a/hosts/idols-aquamarine/asr.nix +++ b/hosts/idols-aquamarine/asr.nix @@ -1,5 +1,6 @@ { lib, + mylib, myvars, pkgs, pkgs-npu, @@ -86,23 +87,26 @@ in ZE_ENABLE_ALT_DRIVERS = "${lib.getLib pkgs-npu.intel-npu-driver}/lib/libze_intel_npu.so"; }; - serviceConfig = { + # Reads its model, writes its cache, uses the NPU and listens on the LAN. + serviceConfig = mylib.systemdHardening.service // { ExecStart = "${pythonEnv}/bin/python ${./asr/server.py}"; CacheDirectory = "whisper-asr"; DynamicUser = true; Restart = "on-failure"; RestartSec = 5; - # The service only reads its model, writes its cache and listens on the - # LAN; it needs no other part of the filesystem or any privilege. - NoNewPrivileges = true; - PrivateTmp = true; - ProtectHome = true; - ProtectSystem = "strict"; RestrictAddressFamilies = [ "AF_INET" "AF_UNIX" ]; + # The NPU is /dev/accel/accel0; allow that device class only. + PrivateDevices = false; + DevicePolicy = "closed"; + DeviceAllow = [ "char-accel rw" ]; + # OpenVINO's oneDNN kernels and libffi closures generate code at runtime. + MemoryDenyWriteExecute = false; + # OpenVINO sizes its thread pools from /proc/cpuinfo and /proc/meminfo. + ProcSubset = "all"; }; }; } diff --git a/hosts/idols-aquamarine/default.nix b/hosts/idols-aquamarine/default.nix index 362f9355..f2c91e51 100644 --- a/hosts/idols-aquamarine/default.nix +++ b/hosts/idols-aquamarine/default.nix @@ -58,6 +58,7 @@ in # Local btrfs snapshots of @persistent (SSH host keys, /etc/secureboot, the # NixOS state). Same layout as the other physical hosts; no off-host target. modules.btrbk.enable = true; + modules.security.audit.enable = true; # smartd autodetects devices and logs to the journal; the smartctl exporter # (modules/nixos/base/monitoring.nix) is what feeds wear and media errors to @@ -81,7 +82,12 @@ in environment.sessionVariables.ZE_ENABLE_ALT_DRIVERS = "${lib.getLib pkgs-npu.intel-npu-driver}/lib/libze_intel_npu.so"; # The daemons of the hosts that build here (modules/nixos/base/remote-builders.nix). - users.users.root.openssh.authorizedKeys.keys = myvars.remoteBuilderClientKeys; + # Their host keys have no passphrase, so they only get the daemon protocol, as + # nix.sshServe does. `--store daemon` hands builds to the system daemon instead + # of running them inside this audited login session (modules/nixos/base/auditd.nix). + users.users.root.openssh.authorizedKeys.keys = map ( + key: ''restrict,command="nix-daemon --stdio --store daemon" ${key}'' + ) myvars.remoteBuilderClientKeys; # systemd-journal - cap disk usage # https://www.freedesktop.org/software/systemd/man/latest/journald.conf.html diff --git a/lib/README.md b/lib/README.md index 93f11fab..e6725209 100644 --- a/lib/README.md +++ b/lib/README.md @@ -21,10 +21,14 @@ reduce code duplication and make it easier to add new machines. 7. **`genMicrovmGuestModule.nix`** - NixOS microVM guest configuration generator 8. **`genLibvirtDomainXml.nix`** - libvirt domain XML generator (imported directly by host configs) +### Shared Presets + +9. **`systemdHardening.nix`** - systemd sandboxing preset for exposed services + ### Entry Point -9. **`default.nix`** - Main entry point that imports the generators and exports them as a single - attribute set, plus the `relativeToRoot` and `scanPaths` helpers +10. **`default.nix`** - Main entry point that imports the generators and exports them as a single + attribute set, plus the `relativeToRoot` and `scanPaths` helpers ## Usage diff --git a/lib/default.nix b/lib/default.nix index 154dc5b0..45115d01 100644 --- a/lib/default.nix +++ b/lib/default.nix @@ -9,6 +9,8 @@ genVmHostModule = import ./genVmHostModule.nix; genMicrovmGuestModule = import ./genMicrovmGuestModule.nix; + systemdHardening = import ./systemdHardening.nix { inherit lib; }; + # use path relative to the root of the project relativeToRoot = lib.path.append ../.; scanPaths = diff --git a/lib/systemdHardening.nix b/lib/systemdHardening.nix new file mode 100644 index 00000000..5d333dad --- /dev/null +++ b/lib/systemdHardening.nix @@ -0,0 +1,47 @@ +# Shared systemd sandboxing preset, adapted from xddxdd's serviceHardenArgs +# (github.com/xddxdd/nixos-config, nixos/minimal-modules/harden.nix). +{ lib }: +rec { + # For a network daemon that runs as an unprivileged user and only needs its + # own state; override keys where a service needs more. + service = { + AmbientCapabilities = ""; + CapabilityBoundingSet = ""; + LockPersonality = true; + MemoryDenyWriteExecute = true; + NoNewPrivileges = true; + PrivateDevices = true; + PrivateMounts = true; + PrivateTmp = true; + ProcSubset = "pid"; + ProtectClock = true; + ProtectControlGroups = true; + ProtectHome = true; + ProtectHostname = true; + ProtectKernelLogs = true; + ProtectKernelModules = true; + ProtectKernelTunables = true; + ProtectProc = "invisible"; + ProtectSystem = "strict"; + RemoveIPC = true; + RestrictAddressFamilies = [ + "AF_UNIX" + "AF_INET" + "AF_INET6" + ]; + RestrictNamespaces = true; + RestrictRealtime = true; + RestrictSUIDSGID = true; + SystemCallArchitectures = "native"; + # Fail a denied syscall rather than kill the service while it is new to + # the preset; switch to "kill" once it has run stably. + SystemCallErrorNumber = "EPERM"; + SystemCallFilter = [ + "@system-service" + "~@clock @cpu-emulation @debug @module @mount @obsolete @privileged @raw-io @reboot @swap" + ]; + }; + + # Apply a preset at mkOptionDefault priority, so nixpkgs module values win. + mkDefaults = lib.mapAttrs (_: lib.mkOptionDefault); +} diff --git a/modules/README.md b/modules/README.md index 0d3e9233..ab9b3cbd 100644 --- a/modules/README.md +++ b/modules/README.md @@ -27,6 +27,7 @@ modules/ │ └── users.nix # macOS user management └── nixos/ # NixOS-specific modules ├── base/ # Base NixOS configuration + │ ├── auditd.nix # Audit trail for root sessions and credentials │ ├── btop.nix # System resource monitor │ ├── btrbk.nix # Local btrfs snapshots │ ├── core.nix # Core system settings diff --git a/modules/nixos/base/auditd.nix b/modules/nixos/base/auditd.nix new file mode 100644 index 00000000..45ce14e5 --- /dev/null +++ b/modules/nixos/base/auditd.nix @@ -0,0 +1,128 @@ +{ + config, + lib, + options, + pkgs, + ... +}: +let + cfg = config.modules.security.audit; + inherit (config.services) openssh; + + # Audit trail for login sessions, so that a stolen deployment key leaves a + # record of what each session did; see SECURITY.md ("Audit trail"). + # `auditctl -R` aborts at the first watch whose path is missing, leaving + # auditing off, so every watched path must exist when the rules load. + + # Secrets are symlinks into a per-deploy generation under the agenix ramfs, + # which a per-file watch would lose on the next deploy; watch the ramfs tree. + agenixSecrets = lib.optionals (options ? age) (lib.attrValues config.age.secrets); + hasAgenix = agenixSecrets != [ ]; + credentialDirs = lib.optional hasAgenix config.age.secretsMountPoint; + credentialFiles = lib.unique ( + map (secret: secret.path) (lib.filter (secret: !secret.symlink) agenixSecrets) + # The host key is also agenix's identity: reading it decrypts every secret. + ++ lib.optionals openssh.enable (map (key: key.path) openssh.hostKeys) + ++ lib.optionals hasAgenix config.age.identityPaths + ); + + identityFiles = map (name: "/etc/${name}") [ + "passwd" + "shadow" + "group" + "gshadow" + "sudoers" + ]; + + watchedDirs = credentialDirs ++ [ + "/etc/ssh" + "/etc/pam.d" + "/etc/nix" + "/etc/audit" + "/var/log/audit" + ]; + watchedFiles = credentialFiles ++ identityFiles ++ [ "/nix/var/nix/profiles/system" ]; + + watchFile = + key: perm: path: + "-a always,exit -F path=${path} -F perm=${perm} -k ${key}"; + watchDir = + key: perm: dir: + "-a always,exit -F dir=${dir} -F perm=${perm} -k ${key}"; + # A b64-only syscall rule misses 32-bit binaries. + arches = [ "b64" ] ++ lib.optional pkgs.stdenv.hostPlatform.isx86_64 "b32"; + perArch = rule: map (arch: "-a always,exit -F arch=${arch} ${rule}") arches; +in +{ + options.modules.security.audit.enable = lib.mkEnableOption '' + the Linux audit trail (auditd) for login sessions: every command run as + root, access to decrypted secrets and SSH host keys, and changes to SSH, + account, PAM, Nix trust and audit configuration. Logs stay in + /var/log/audit on the host + ''; + + config = lib.mkIf cfg.enable { + assertions = [ + { + # auditctl -R splits fields on whitespace; it does not shell-unquote paths. + assertion = lib.all (path: builtins.match ".*[[:space:]].*" path == null) ( + watchedDirs ++ watchedFiles + ); + message = "modules.security.audit: audited paths cannot contain whitespace in the native auditctl rule file."; + } + ]; + + security.auditd.enable = true; + security.audit = { + enable = true; + backlogLimit = lib.mkDefault 8192; + # The first matching exit rule sets the key: specific watches go first. + rules = lib.mkBefore ( + # Evaluated at fork: tasks without a login uid (everything systemd + # starts) get no audit context, so no per-syscall cost. + [ "-a never,task -F auid=unset" ] + ++ map (watchFile "credential-access" "rwa") credentialFiles + ++ map (watchDir "credential-access" "rwa") credentialDirs + ++ [ (watchDir "ssh-config" "wa" "/etc/ssh") ] + ++ map (watchFile "identity" "wa") identityFiles + ++ [ + (watchDir "identity" "wa" "/etc/pam.d") + (watchDir "nix-trust" "wa" "/etc/nix") + (watchFile "nix-trust" "wa" "/nix/var/nix/profiles/system") + (watchDir "audit-config" "wa" "/etc/audit") + (watchDir "audit-config" "wa" "/var/log/audit") + ] + ++ perArch "-S init_module,finit_module,delete_module -k kernel-module" + ++ perArch "-S execve,execveat -F euid=0 -F auid!=unset -k root-session" + ++ [ + "-a always,exit -S all -F exit=-EACCES -F auid!=unset -k access-denied" + "-a always,exit -S all -F exit=-EPERM -F auid!=unset -k access-denied" + ] + ); + }; + + security.auditd.settings = { + log_file = "/var/log/audit/audit.log"; + max_log_file_action = "rotate"; + max_log_file = 50; # MiB + num_logs = 5; + }; + + # Parents of watches that may not exist yet on a fresh system. + systemd.tmpfiles.rules = [ + "d /var/log/audit 0700 root root -" + "d /nix/var/nix/profiles 0755 root root -" + ]; + + # A watch binds to the inode present at load time, so load after the + # bind mounts (preservation), tmpfiles and the agenix ramfs. + systemd.services.audit-rules-nixos = { + after = [ + "local-fs.target" + "systemd-tmpfiles-setup.service" + ] + ++ lib.optional hasAgenix "agenix-install-secrets.service"; + unitConfig.RequiresMountsFor = lib.unique (watchedDirs ++ map dirOf watchedFiles); + }; + }; +} diff --git a/modules/nixos/base/ssh.nix b/modules/nixos/base/ssh.nix index 28d77a6a..656d5e37 100644 --- a/modules/nixos/base/ssh.nix +++ b/modules/nixos/base/ssh.nix @@ -14,8 +14,12 @@ # root login stays available for Colmena/MicroVM deployment, keys only PermitRootLogin = lib.mkDefault "prohibit-password"; PasswordAuthentication = false; # disable password login + # nixpkgs defaults it to true, and PAM then still accepts the Unix password + KbdInteractiveAuthentication = false; }; openFirewall = true; + # Only keys declared in Nix log in; a hand- or intruder-added key is ignored + authorizedKeysInHomedir = false; }; # NixOS appends an `Include` of systemd's ssh_config.d drop-in to diff --git a/outputs/aarch64-linux/tests/security-audit/expected.nix b/outputs/aarch64-linux/tests/security-audit/expected.nix new file mode 100644 index 00000000..bda15660 --- /dev/null +++ b/outputs/aarch64-linux/tests/security-audit/expected.nix @@ -0,0 +1,2 @@ +{ lib, outputs }: +import ../../../x86_64-linux/tests/security-audit/expected.nix { inherit lib outputs; } diff --git a/outputs/aarch64-linux/tests/security-audit/expr.nix b/outputs/aarch64-linux/tests/security-audit/expr.nix new file mode 100644 index 00000000..cdee96f2 --- /dev/null +++ b/outputs/aarch64-linux/tests/security-audit/expr.nix @@ -0,0 +1,2 @@ +{ lib, outputs }: +import ../../../x86_64-linux/tests/security-audit/expr.nix { inherit lib outputs; } diff --git a/outputs/aarch64-linux/tests/security-ssh-auth/expected.nix b/outputs/aarch64-linux/tests/security-ssh-auth/expected.nix new file mode 100644 index 00000000..cece43b6 --- /dev/null +++ b/outputs/aarch64-linux/tests/security-ssh-auth/expected.nix @@ -0,0 +1,25 @@ +{ + lib, + outputs, + myvars, +}: +lib.genAttrs (builtins.attrNames outputs.nixosConfigurations) ( + name: + { + # Neither password nor PAM keyboard-interactive login is offered. + PasswordAuthentication = false; + KbdInteractiveAuthentication = false; + # Undeclared keys in ~/.ssh/authorized_keys are not honoured. + authorizedKeysInHomedir = false; + # Only the interactive admin keys open a root shell; remote-builder host keys + # are pinned to the nix-daemon protocol, served by the system daemon. + unrestrictedRootKeys = builtins.length ( + myvars.mainSshAuthorizedKeys ++ myvars.secondaryAuthorizedKeys + ); + } + // lib.optionalAttrs (name == "aquamarine") { + builderKeys = map ( + key: ''restrict,command="nix-daemon --stdio --store daemon" ${key}'' + ) myvars.remoteBuilderClientKeys; + } +) diff --git a/outputs/aarch64-linux/tests/security-ssh-auth/expr.nix b/outputs/aarch64-linux/tests/security-ssh-auth/expr.nix new file mode 100644 index 00000000..591d56d7 --- /dev/null +++ b/outputs/aarch64-linux/tests/security-ssh-auth/expr.nix @@ -0,0 +1,21 @@ +{ + lib, + outputs, +}: +lib.genAttrs (builtins.attrNames outputs.nixosConfigurations) ( + name: + let + inherit (outputs.nixosConfigurations.${name}.config) services users; + in + { + inherit (services.openssh.settings) PasswordAuthentication KbdInteractiveAuthentication; + inherit (services.openssh) authorizedKeysInHomedir; + # Keys that may open a root session without a forced command. + unrestrictedRootKeys = builtins.length ( + lib.filter (key: !lib.hasPrefix "restrict," key) users.users.root.openssh.authorizedKeys.keys + ); + } + // lib.optionalAttrs (name == "aquamarine") { + builderKeys = lib.filter (lib.hasPrefix "restrict,") users.users.root.openssh.authorizedKeys.keys; + } +) diff --git a/outputs/default.nix b/outputs/default.nix index b390ed0f..79f04f83 100644 --- a/outputs/default.nix +++ b/outputs/default.nix @@ -214,6 +214,12 @@ in pkgs = nixpkgs.legacyPackages.${system}; networking = myvars.networking; }; + # Load the audit rules in a VM: one rule that fails to load leaves + # auditing off, which evaluation cannot catch. + security-auditd = import ../tests/security-auditd.nix { + pkgs = nixpkgs.legacyPackages.${system}; + inherit (inputs) agenix; + }; } ); diff --git a/outputs/x86_64-linux/tests/security-audit/expected.nix b/outputs/x86_64-linux/tests/security-audit/expected.nix new file mode 100644 index 00000000..adccbc45 --- /dev/null +++ b/outputs/x86_64-linux/tests/security-audit/expected.nix @@ -0,0 +1,24 @@ +{ + lib, + outputs, +}: +lib.genAttrs (builtins.attrNames outputs.nixosConfigurations) ( + name: + let + # Homelab servers reachable with the root deployment keys; not desktops. + audited = builtins.elem name [ + "youko" + "shoryu" + "shushou" + "aquamarine" + ]; + in + { + auditd = audited; + rootSession = audited; + sessionsOnly = audited; + # Only emitted where agenix has secrets: a watch on a missing path + # aborts loading the remaining rules. + agenixWatch = name == "youko"; + } +) diff --git a/outputs/x86_64-linux/tests/security-audit/expr.nix b/outputs/x86_64-linux/tests/security-audit/expr.nix new file mode 100644 index 00000000..972b134d --- /dev/null +++ b/outputs/x86_64-linux/tests/security-audit/expr.nix @@ -0,0 +1,16 @@ +{ + lib, + outputs, +}: +lib.genAttrs (builtins.attrNames outputs.nixosConfigurations) ( + name: + let + inherit (outputs.nixosConfigurations.${name}) config; + in + { + auditd = config.security.auditd.enable; + rootSession = lib.any (lib.hasSuffix "-k root-session") config.security.audit.rules; + sessionsOnly = lib.elem "-a never,task -F auid=unset" config.security.audit.rules; + agenixWatch = lib.elem "-a always,exit -F dir=/run/agenix.d -F perm=rwa -k credential-access" config.security.audit.rules; + } +) diff --git a/outputs/x86_64-linux/tests/security-service-hardening/expected.nix b/outputs/x86_64-linux/tests/security-service-hardening/expected.nix new file mode 100644 index 00000000..246fc288 --- /dev/null +++ b/outputs/x86_64-linux/tests/security-service-hardening/expected.nix @@ -0,0 +1,24 @@ +let + sandbox = { + CapabilityBoundingSet = ""; + NoNewPrivileges = true; + PrivateDevices = true; + ProtectSystem = "strict"; + SystemCallFilter = [ + "@system-service" + "~@clock @cpu-emulation @debug @module @mount @obsolete @privileged @raw-io @reboot @swap" + ]; + }; +in +{ + # Only 80/443. + caddy = sandbox // { + CapabilityBoundingSet = [ "CAP_NET_BIND_SERVICE" ]; + }; + rustfs = sandbox; + vmalert-homelab = sandbox; + # The NPU is allowed through DeviceAllow instead. + whisper-asr = sandbox // { + PrivateDevices = false; + }; +} diff --git a/outputs/x86_64-linux/tests/security-service-hardening/expr.nix b/outputs/x86_64-linux/tests/security-service-hardening/expr.nix new file mode 100644 index 00000000..cf7acb9b --- /dev/null +++ b/outputs/x86_64-linux/tests/security-service-hardening/expr.nix @@ -0,0 +1,21 @@ +{ lib, outputs }: +let + keys = [ + "CapabilityBoundingSet" + "NoNewPrivileges" + "PrivateDevices" + "ProtectSystem" + "SystemCallFilter" + ]; + units = + host: names: + lib.genAttrs names ( + n: lib.getAttrs keys outputs.nixosConfigurations.${host}.config.systemd.services.${n}.serviceConfig + ); +in +units "youko" [ + "caddy" + "rustfs" + "vmalert-homelab" +] +// units "aquamarine" [ "whisper-asr" ] diff --git a/outputs/x86_64-linux/tests/security-ssh-auth/expected.nix b/outputs/x86_64-linux/tests/security-ssh-auth/expected.nix new file mode 100644 index 00000000..cece43b6 --- /dev/null +++ b/outputs/x86_64-linux/tests/security-ssh-auth/expected.nix @@ -0,0 +1,25 @@ +{ + lib, + outputs, + myvars, +}: +lib.genAttrs (builtins.attrNames outputs.nixosConfigurations) ( + name: + { + # Neither password nor PAM keyboard-interactive login is offered. + PasswordAuthentication = false; + KbdInteractiveAuthentication = false; + # Undeclared keys in ~/.ssh/authorized_keys are not honoured. + authorizedKeysInHomedir = false; + # Only the interactive admin keys open a root shell; remote-builder host keys + # are pinned to the nix-daemon protocol, served by the system daemon. + unrestrictedRootKeys = builtins.length ( + myvars.mainSshAuthorizedKeys ++ myvars.secondaryAuthorizedKeys + ); + } + // lib.optionalAttrs (name == "aquamarine") { + builderKeys = map ( + key: ''restrict,command="nix-daemon --stdio --store daemon" ${key}'' + ) myvars.remoteBuilderClientKeys; + } +) diff --git a/outputs/x86_64-linux/tests/security-ssh-auth/expr.nix b/outputs/x86_64-linux/tests/security-ssh-auth/expr.nix new file mode 100644 index 00000000..591d56d7 --- /dev/null +++ b/outputs/x86_64-linux/tests/security-ssh-auth/expr.nix @@ -0,0 +1,21 @@ +{ + lib, + outputs, +}: +lib.genAttrs (builtins.attrNames outputs.nixosConfigurations) ( + name: + let + inherit (outputs.nixosConfigurations.${name}.config) services users; + in + { + inherit (services.openssh.settings) PasswordAuthentication KbdInteractiveAuthentication; + inherit (services.openssh) authorizedKeysInHomedir; + # Keys that may open a root session without a forced command. + unrestrictedRootKeys = builtins.length ( + lib.filter (key: !lib.hasPrefix "restrict," key) users.users.root.openssh.authorizedKeys.keys + ); + } + // lib.optionalAttrs (name == "aquamarine") { + builderKeys = lib.filter (lib.hasPrefix "restrict,") users.users.root.openssh.authorizedKeys.keys; + } +) diff --git a/tests/security-auditd.nix b/tests/security-auditd.nix new file mode 100644 index 00000000..5b0f116c --- /dev/null +++ b/tests/security-auditd.nix @@ -0,0 +1,100 @@ +{ pkgs, agenix }: +let + # Throwaway age identity and secret, generated at build time for the VM. + testSecrets = pkgs.runCommand "auditd-test-secrets" { nativeBuildInputs = [ pkgs.age ]; } '' + mkdir $out + age-keygen -o $out/key.txt + echo not-a-real-secret | age -r "$(age-keygen -y $out/key.txt)" -o $out/secret.age + ''; +in +pkgs.testers.runNixOSTest { + name = "security-auditd"; + nodes.machine = + { config, ... }: + { + imports = [ + ../modules/nixos/base/auditd.nix + agenix.nixosModules.default + ]; + modules.security.audit.enable = true; + services.openssh.enable = true; + age.identityPaths = [ "${testSecrets}/key.txt" ]; + age.secrets.test-secret.file = "${testSecrets}/secret.age"; + users.users.alice.isNormalUser = true; + environment.etc."audit-rule-count".text = toString (builtins.length config.security.audit.rules); + virtualisation.memorySize = 1024; + }; + testScript = '' + import shlex + + def ssh(cmd): + return machine.execute( + "ssh -i /root/id -o StrictHostKeyChecking=no root@localhost " + shlex.quote(cmd) + ) + + # auditd writes asynchronously, so poll; `-ul 0` keeps records whose + # login uid (auid) is root, i.e. from the SSH session. Without + # --input-logs, ausearch reads stdin when it is not a terminal. + def wait_logged(key, match, flags="", count=1): + machine.wait_until_succeeds( + f"test $(ausearch --input-logs -i -k {key} {flags} | grep -c -- {shlex.quote(match)}) -ge {count}" + ) + + machine.wait_for_unit("audit-rules-nixos.service") + machine.wait_for_unit("auditd.service") + machine.wait_for_unit("sshd.service") + + with subtest("every rule loads and auditing is enabled"): + t.assertIn("enabled 1", machine.succeed("auditctl -s")) + rules = machine.succeed("auditctl -l") + t.assertEqual(len(rules.splitlines()), int(machine.succeed("cat /etc/audit-rule-count"))) + t.assertIn("-w /run/agenix.d -p rwa -k credential-access", rules) + t.assertIn("-a never,task -F auid=-1", rules) + + # The test driver's shell is a service: this setup is not audited. + machine.succeed( + "ssh-keygen -q -t ed25519 -N ''' -f /root/id", + "install -d -m 700 /root/.ssh", + "cat /root/id.pub >> /root/.ssh/authorized_keys", + ) + + with subtest("an SSH root session logs its commands and secret reads"): + ssh("cat /run/agenix/test-secret; ${pkgs.hello}/bin/hello") + wait_logged("root-session", "hello", "-ul 0") + # sshd is built without libaudit; PAM records the login and its address. + machine.succeed("ausearch --input-logs -i -m USER_START -ul 0 | grep -q 'addr=::1 terminal=ssh'") + wait_logged("credential-access", "name=/run/agenix/test-secret", "-ul 0") + + with subtest("a service is not audited; the same actions from a session are"): + machine.succeed( + "systemd-run --wait -p Environment=PATH=${pkgs.coreutils}/bin ${pkgs.runtimeShell} -c " + "'cat /run/agenix/test-secret >/dev/null; touch /etc/ssh/from-service'" + ) + ssh("cat /run/agenix/test-secret >/dev/null; touch /etc/ssh/from-session") + wait_logged("ssh-config", "from-session", "-ul 0") + wait_logged("credential-access", "name=/run/agenix/test-secret", "-ul 0", count=2) + machine.succeed("test -e /etc/ssh/from-service") + machine.fail("ausearch --input-logs -i -k ssh-config | grep -q from-service") + # auditctl's own rule loads also carry the key (CONFIG_CHANGE, no auid). + machine.fail("ausearch --input-logs -i -k credential-access | grep auid=unset | grep -q comm=cat") + machine.fail("ausearch --input-logs -i -k root-session | grep -q from-service") + # systemd-executor opens the agenix mount point with O_PATH while it + # builds a sandboxed unit's namespace; it has no auid either. + machine.succeed("systemctl restart nscd") + machine.fail("ausearch --input-logs -i -k credential-access | grep -q comm=.nsncd") + + with subtest("the agenix watch survives a new secrets generation"): + machine.succeed("/run/current-system/activate") + machine.succeed("readlink /run/agenix | grep -q '/2$'") + ssh("cat /run/agenix/test-secret >/dev/null") + wait_logged("credential-access", "name=/run/agenix/test-secret", "-ul 0", count=3) + + with subtest("denials in a login session are recorded"): + ssh("su alice -s /bin/sh -c 'cat /etc/shadow'") + wait_logged("access-denied", "shadow", "-ul 0") + + with subtest("auditd keeps its log; the journal keeps a copy"): + machine.succeed("test -s /var/log/audit/audit.log") + machine.succeed("journalctl _TRANSPORT=audit -q | grep -q root-session") + ''; +}