diff --git a/SECURITY.md b/SECURITY.md index e61f5bb7..75e29648 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -33,18 +33,18 @@ small, reversible controls and narrow exceptions; benchmark before adopting expe ## Architecture and trust boundaries -| Boundary | Current design | Limitation | -| --------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| Administration | Key-based SSH; root key login retained for Colmena/MicroVM deployment | Deployment credentials are root-equivalent; password login being disabled does not restrict stolen keys | -| Host network | Shared nftables firewall trusts the home LAN, tailnet interface and local Podman bridge; IPv6 link-local is accepted | These are broad trust zones, not service-level least privilege; router exposure and tailnet ACLs require separate verification | -| Monitoring | node/smartctl exporters bind the host LAN IPv4 (9100/9633); firewall rules restrict exporter ports 9100, 9835 and 9633 to the monitoring host over IPv4 | nvidia_gpu_exporter (9835) still binds wildcard; the IPv6 denial for those three ports is in the shared firewall; current running hosts require separate verification | -| VM hosts | shoryu, shushou and youko run guests behind `br0`; ruby/kana are libvirt guests | A bridge is not network isolation; host compromise affects its guests | -| k3s | Three control-plane and three worker MicroVMs; Cilium, API VIP and NFS CSI | Cluster-admin, privileged/host-mounted pods and writable NFS data are powerful trust boundaries | -| MicroVM storage | Host store shared read-only through virtiofs; guest `/etc`, `/var` and `/home` persisted in separate images | Read-only store does not isolate the host-side virtiofs implementation or protect writable guest state | -| Desktop apps | Selected apps use nixpak/bubblewrap; Podman is rootless | Rules and shared directories vary per app; user namespaces are needed for these sandboxes | -| Secrets | agenix declarations here, encrypted data in the private nix-secrets repository | Recipients and plaintext file permissions must be reviewed; root on a recipient can read its decrypted secrets | -| Supply chain | Locked flake inputs; explicit binary-cache trust keys on NixOS; normal user is not a Nix trusted-user | A trusted cache or imported Nix module can supply code executed as root; a lock file is not a security audit | -| Recovery | NixOS generations, btrbk/restic, hourly read-only snapshots of the REST store, and documented backup procedures | A rollback does not undo compromised data or secrets; backup configuration alone does not prove restore works; the offsite immutable copy is still planned | +| Boundary | Current design | Limitation | +| --------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Administration | Key-based SSH; root key login retained for Colmena/MicroVM deployment; only keys declared in Nix are honoured, and no password or keyboard-interactive login; remote-builder host keys only reach `nix-daemon` | Deployment credentials are root-equivalent file keys, usable by any same-user process through the desktop SSH agent; one stolen key reaches every host | +| Host network | Shared nftables firewall trusts the home LAN, tailnet interface and local Podman bridge; IPv6 link-local is accepted | These are broad trust zones, not service-level least privilege; router exposure and tailnet ACLs require separate verification | +| Monitoring | node/smartctl exporters bind the host LAN IPv4 (9100/9633); firewall rules restrict exporter ports 9100, 9835 and 9633 to the monitoring host over IPv4 | nvidia_gpu_exporter (9835) still binds wildcard; the IPv6 denial for those three ports is in the shared firewall; current running hosts require separate verification | +| VM hosts | shoryu, shushou and youko run guests behind `br0`; ruby/kana are libvirt guests | A bridge is not network isolation; host compromise affects its guests | +| k3s | Three control-plane and three worker MicroVMs; Cilium, API VIP and NFS CSI | Cluster-admin, privileged/host-mounted pods and writable NFS data are powerful trust boundaries | +| MicroVM storage | Host store shared read-only through virtiofs; guest `/etc`, `/var` and `/home` persisted in separate images | Read-only store does not isolate the host-side virtiofs implementation or protect writable guest state | +| Desktop apps | Selected apps use nixpak/bubblewrap; Podman is rootless | Rules and shared directories vary per app; user namespaces are needed for these sandboxes | +| Secrets | agenix declarations here, encrypted data in the private nix-secrets repository | Recipients and plaintext file permissions must be reviewed; root on a recipient can read its decrypted secrets | +| Supply chain | Locked flake inputs; explicit binary-cache trust keys on NixOS; normal user is not a Nix trusted-user | A trusted cache or imported Nix module can supply code executed as root; a lock file is not a security audit | +| Recovery | NixOS generations, btrbk/restic, hourly read-only snapshots of the REST store, and documented backup procedures | A rollback does not undo compromised data or secrets; backup configuration alone does not prove restore works; the offsite immutable copy is still planned | Sources: [host layout](hosts/README.md), [k3s deployment](hosts/k8s/README.md), [firewall](modules/nixos/base/networking/firewall.nix), [SSH](modules/nixos/base/ssh.nix), @@ -104,11 +104,35 @@ an advisory and affected kernel versions before adding CVE claims; search snippe - AppArmor is enabled, but the only explicit policy, sudo's, is in complain mode. Complain mode logs rather than blocks. Existing FHS-oriented abstractions need Nix store path review; this is not broad enforced application confinement. -- Systemd service sandboxing is per-service, not a global policy. Review capabilities, writable - paths, network access and syscall requirements before tightening an individual service. +- Systemd service sandboxing is per-service, not a global policy. The exposed services nixpkgs + leaves weakly sandboxed (caddy, rustfs, vmalert, whisper-asr) use the preset in + [systemdHardening.nix](lib/systemdHardening.nix). Review capabilities, writable paths, network + access and syscall requirements before tightening an individual service. See [application hardening](hardening/README.md) for wrapper details. +### Audit trail + +`modules.security.audit` ([auditd.nix](modules/nixos/base/auditd.nix)) runs auditd on youko, shoryu, +shushou and aquamarine. It records, for login sessions only (SSH, and sudo or su from one): every +root command (`root-session`), access to decrypted secrets and SSH host keys (`credential-access`), +writes to SSH, account/PAM, Nix trust and audit configuration, module loads, and denials. + +- Services are not audited: a `never,task` rule gives tasks without a login uid no audit context, so + virtiofsd, hypervisors, NFS, PostgreSQL and Nix builds skip the per-syscall audit path. A + compromised service is therefore invisible until it opens a login session. +- Logs stay on the host and root can erase them; shipping them off-host is a P1 TODO. +- One watch on a missing path makes `auditctl -R` stop and leaves auditing off; + `nix build .#checks.x86_64-linux.security-auditd` loads the rules in a VM. + +Query as root; sshd is built without libaudit, so use PAM's `USER_START` records for logins: + +```sh +ausearch -i -m USER_START --start today # logins with address and session id +ausearch -i -k root-session --start today # root commands, with auid and ses +ausearch -i --session # everything one session did +``` + ## Updating, deploying and recovering 1. Check the advisory and affected package/kernel; prioritize active exploitation and reachable @@ -355,6 +379,8 @@ separate verification. | Priority | Work | Acceptance / constraints | | -------- | -------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| P1 | Move root/deploy SSH to hardware-backed keys | `sk-ssh-ed25519` keys with touch for root, with SSH multiplexing to keep Colmena usable; an agent lifetime or confirmation; a tested break-glass path | +| P1 | Alert on SSH logins and ship auth/audit logs off-host | Off-host copy host root cannot rewrite; alert on an unknown key or source, a session-rate spike and `root-session`/`credential-access`; redact key material | | P1 | Roll out and verify the implemented baseline/exporter rules | Runtime sysctl readback, allowed IPv4 scrape and denied IPv6/non-monitoring scrape tests; preserve Nix/Podman/Cilium/NFS functionality | | P1 | Review router mappings and tailnet ACLs; narrow service exposure where warranted | Audit was limited to sampled host listeners; test intended and denied access without locking out deployment | | P1 | Review NFS client scope and root identity requirements | Restrict to required VM hosts/CSI nodes where practical; test provisioning, permissions and existing PVCs before changing squash semantics | diff --git a/hosts/12kingdoms-shoryu/default.nix b/hosts/12kingdoms-shoryu/default.nix index dc513c8d..f300a827 100644 --- a/hosts/12kingdoms-shoryu/default.nix +++ b/hosts/12kingdoms-shoryu/default.nix @@ -25,6 +25,7 @@ in ]; modules.btrbk.enable = true; + modules.security.audit.enable = true; # MoreFine S500Plus: Radeon iGPU. The ROCm build of btop shows the GPU panel; # node_exporter's drm collector exposes its utilization/VRAM. diff --git a/hosts/12kingdoms-shushou/default.nix b/hosts/12kingdoms-shushou/default.nix index 1957e286..d04f10db 100644 --- a/hosts/12kingdoms-shushou/default.nix +++ b/hosts/12kingdoms-shushou/default.nix @@ -24,6 +24,7 @@ in ]; modules.btrbk.enable = true; + modules.security.audit.enable = true; # MinisForum UM560: Radeon iGPU. The ROCm build of btop shows the GPU panel; # node_exporter's drm collector exposes its utilization/VRAM. diff --git a/hosts/12kingdoms-youko/default.nix b/hosts/12kingdoms-youko/default.nix index f33ec3c0..c3095f12 100644 --- a/hosts/12kingdoms-youko/default.nix +++ b/hosts/12kingdoms-youko/default.nix @@ -58,6 +58,7 @@ in }; modules.btrbk.enable = true; + modules.security.audit.enable = true; # HDD tuning/diagnostics for the two spinning disks behind the USB-SATA bridge. environment.systemPackages = [ pkgs.hdparm ]; diff --git a/hosts/12kingdoms-youko/homelab-services/caddy.nix b/hosts/12kingdoms-youko/homelab-services/caddy.nix index 02a08cd8..b56f2413 100644 --- a/hosts/12kingdoms-youko/homelab-services/caddy.nix +++ b/hosts/12kingdoms-youko/homelab-services/caddy.nix @@ -232,11 +232,22 @@ in }; }; - # caddy loads the lego-issued files at startup, so wait for the acme service; - # otherwise its config fails to load before the cert exists. systemd.services.caddy = { + # caddy loads the lego-issued files at startup, so wait for the acme + # service; otherwise its config fails to load before the cert exists. wants = [ "acme-writefor.fun.service" ]; after = [ "acme-writefor.fun.service" ]; + + # Sandbox the LAN front door; it writes only its State/LogsDirectory. + serviceConfig = mylib.systemdHardening.mkDefaults mylib.systemdHardening.service // { + # Only 80/443. No CAP_NET_ADMIN: the module already raises the UDP buffer + # sysctls quic-go would force. "" resets the package unit's list. + AmbientCapabilities = [ + "" + "CAP_NET_BIND_SERVICE" + ]; + CapabilityBoundingSet = [ "CAP_NET_BIND_SERVICE" ]; + }; }; # caddy is reached from the LAN/tailnet only; the shared firewall diff --git a/hosts/12kingdoms-youko/homelab-services/gitea.nix b/hosts/12kingdoms-youko/homelab-services/gitea.nix index 10f022f7..b0e6aaef 100644 --- a/hosts/12kingdoms-youko/homelab-services/gitea.nix +++ b/hosts/12kingdoms-youko/homelab-services/gitea.nix @@ -1,5 +1,12 @@ { pkgs, ... }: { + # Git over SSH reads Gitea's own ~/.ssh/authorized_keys (keys pinned to + # `gitea serv`), which modules/nixos/base/ssh.nix otherwise ignores. + services.openssh.extraConfig = '' + Match User gitea + AuthorizedKeysFile %h/.ssh/authorized_keys /etc/ssh/authorized_keys.d/%u + ''; + # https://github.com/NixOS/nixpkgs/blob/nixos-26.05/nixos/modules/services/misc/gitea.nix services.gitea = { enable = true; diff --git a/hosts/12kingdoms-youko/homelab-services/monitoring/alert.nix b/hosts/12kingdoms-youko/homelab-services/monitoring/alert.nix index 912ed747..3f092fcb 100644 --- a/hosts/12kingdoms-youko/homelab-services/monitoring/alert.nix +++ b/hosts/12kingdoms-youko/homelab-services/monitoring/alert.nix @@ -1,4 +1,4 @@ -{ config, ... }: +{ config, mylib, ... }: { # https://docs.victoriametrics.com/victoriametrics/vmalert/ services.vmalert.instances."homelab" = { @@ -34,6 +34,9 @@ }; }; + # nixpkgs only sets DynamicUser; vmalert talks to loopback and keeps no state. + systemd.services.vmalert-homelab.serviceConfig = mylib.systemdHardening.mkDefaults mylib.systemdHardening.service; + services.prometheus.alertmanager = { enable = true; listenAddress = "127.0.0.1"; diff --git a/hosts/12kingdoms-youko/homelab-services/rustfs.nix b/hosts/12kingdoms-youko/homelab-services/rustfs.nix index c137e1ee..a637cc0e 100644 --- a/hosts/12kingdoms-youko/homelab-services/rustfs.nix +++ b/hosts/12kingdoms-youko/homelab-services/rustfs.nix @@ -1,7 +1,11 @@ { config, + mylib, ... }: +let + inherit (mylib.systemdHardening) mkDefaults service; +in { # The service module comes from the default nixpkgs (services.rustfs); it # creates the rustfs user/group, the data volume via tmpfiles, and a hardened @@ -26,6 +30,18 @@ environmentFile = config.age.secrets."rustfs.env".path; }; - # The data volume lives on the encrypted HDD subvolume; wait for its mount. - systemd.services.rustfs.unitConfig.RequiresMountsFor = "/data/apps"; + systemd.services.rustfs = { + # The data volume lives on the encrypted HDD subvolume; wait for its mount. + unitConfig.RequiresMountsFor = "/data/apps"; + + # Fill the sandboxing the nixpkgs unit leaves out. + serviceConfig = mkDefaults service // { + # ProtectSystem=strict: the data volume is the only writable path. + ReadWritePaths = [ config.services.rustfs.settings.RUSTFS_VOLUMES ]; + # Its startup endpoint list enumerates the host's interface addresses. + RestrictAddressFamilies = service.RestrictAddressFamilies ++ [ "AF_NETLINK" ]; + # Capacity and observability metrics read host-wide /proc files. + ProcSubset = "all"; + }; + }; } diff --git a/hosts/idols-aquamarine/asr.nix b/hosts/idols-aquamarine/asr.nix index 3c2d9e3a..99d9e393 100644 --- a/hosts/idols-aquamarine/asr.nix +++ b/hosts/idols-aquamarine/asr.nix @@ -1,5 +1,6 @@ { lib, + mylib, myvars, pkgs, pkgs-npu, @@ -86,23 +87,26 @@ in ZE_ENABLE_ALT_DRIVERS = "${lib.getLib pkgs-npu.intel-npu-driver}/lib/libze_intel_npu.so"; }; - serviceConfig = { + # Reads its model, writes its cache, uses the NPU and listens on the LAN. + serviceConfig = mylib.systemdHardening.service // { ExecStart = "${pythonEnv}/bin/python ${./asr/server.py}"; CacheDirectory = "whisper-asr"; DynamicUser = true; Restart = "on-failure"; RestartSec = 5; - # The service only reads its model, writes its cache and listens on the - # LAN; it needs no other part of the filesystem or any privilege. - NoNewPrivileges = true; - PrivateTmp = true; - ProtectHome = true; - ProtectSystem = "strict"; RestrictAddressFamilies = [ "AF_INET" "AF_UNIX" ]; + # The NPU is /dev/accel/accel0; allow that device class only. + PrivateDevices = false; + DevicePolicy = "closed"; + DeviceAllow = [ "char-accel rw" ]; + # OpenVINO's oneDNN kernels and libffi closures generate code at runtime. + MemoryDenyWriteExecute = false; + # OpenVINO sizes its thread pools from /proc/cpuinfo and /proc/meminfo. + ProcSubset = "all"; }; }; } diff --git a/hosts/idols-aquamarine/default.nix b/hosts/idols-aquamarine/default.nix index 362f9355..f2c91e51 100644 --- a/hosts/idols-aquamarine/default.nix +++ b/hosts/idols-aquamarine/default.nix @@ -58,6 +58,7 @@ in # Local btrfs snapshots of @persistent (SSH host keys, /etc/secureboot, the # NixOS state). Same layout as the other physical hosts; no off-host target. modules.btrbk.enable = true; + modules.security.audit.enable = true; # smartd autodetects devices and logs to the journal; the smartctl exporter # (modules/nixos/base/monitoring.nix) is what feeds wear and media errors to @@ -81,7 +82,12 @@ in environment.sessionVariables.ZE_ENABLE_ALT_DRIVERS = "${lib.getLib pkgs-npu.intel-npu-driver}/lib/libze_intel_npu.so"; # The daemons of the hosts that build here (modules/nixos/base/remote-builders.nix). - users.users.root.openssh.authorizedKeys.keys = myvars.remoteBuilderClientKeys; + # Their host keys have no passphrase, so they only get the daemon protocol, as + # nix.sshServe does. `--store daemon` hands builds to the system daemon instead + # of running them inside this audited login session (modules/nixos/base/auditd.nix). + users.users.root.openssh.authorizedKeys.keys = map ( + key: ''restrict,command="nix-daemon --stdio --store daemon" ${key}'' + ) myvars.remoteBuilderClientKeys; # systemd-journal - cap disk usage # https://www.freedesktop.org/software/systemd/man/latest/journald.conf.html diff --git a/lib/README.md b/lib/README.md index 93f11fab..e6725209 100644 --- a/lib/README.md +++ b/lib/README.md @@ -21,10 +21,14 @@ reduce code duplication and make it easier to add new machines. 7. **`genMicrovmGuestModule.nix`** - NixOS microVM guest configuration generator 8. **`genLibvirtDomainXml.nix`** - libvirt domain XML generator (imported directly by host configs) +### Shared Presets + +9. **`systemdHardening.nix`** - systemd sandboxing preset for exposed services + ### Entry Point -9. **`default.nix`** - Main entry point that imports the generators and exports them as a single - attribute set, plus the `relativeToRoot` and `scanPaths` helpers +10. **`default.nix`** - Main entry point that imports the generators and exports them as a single + attribute set, plus the `relativeToRoot` and `scanPaths` helpers ## Usage diff --git a/lib/default.nix b/lib/default.nix index 154dc5b0..45115d01 100644 --- a/lib/default.nix +++ b/lib/default.nix @@ -9,6 +9,8 @@ genVmHostModule = import ./genVmHostModule.nix; genMicrovmGuestModule = import ./genMicrovmGuestModule.nix; + systemdHardening = import ./systemdHardening.nix { inherit lib; }; + # use path relative to the root of the project relativeToRoot = lib.path.append ../.; scanPaths = diff --git a/lib/systemdHardening.nix b/lib/systemdHardening.nix new file mode 100644 index 00000000..5d333dad --- /dev/null +++ b/lib/systemdHardening.nix @@ -0,0 +1,47 @@ +# Shared systemd sandboxing preset, adapted from xddxdd's serviceHardenArgs +# (github.com/xddxdd/nixos-config, nixos/minimal-modules/harden.nix). +{ lib }: +rec { + # For a network daemon that runs as an unprivileged user and only needs its + # own state; override keys where a service needs more. + service = { + AmbientCapabilities = ""; + CapabilityBoundingSet = ""; + LockPersonality = true; + MemoryDenyWriteExecute = true; + NoNewPrivileges = true; + PrivateDevices = true; + PrivateMounts = true; + PrivateTmp = true; + ProcSubset = "pid"; + ProtectClock = true; + ProtectControlGroups = true; + ProtectHome = true; + ProtectHostname = true; + ProtectKernelLogs = true; + ProtectKernelModules = true; + ProtectKernelTunables = true; + ProtectProc = "invisible"; + ProtectSystem = "strict"; + RemoveIPC = true; + RestrictAddressFamilies = [ + "AF_UNIX" + "AF_INET" + "AF_INET6" + ]; + RestrictNamespaces = true; + RestrictRealtime = true; + RestrictSUIDSGID = true; + SystemCallArchitectures = "native"; + # Fail a denied syscall rather than kill the service while it is new to + # the preset; switch to "kill" once it has run stably. + SystemCallErrorNumber = "EPERM"; + SystemCallFilter = [ + "@system-service" + "~@clock @cpu-emulation @debug @module @mount @obsolete @privileged @raw-io @reboot @swap" + ]; + }; + + # Apply a preset at mkOptionDefault priority, so nixpkgs module values win. + mkDefaults = lib.mapAttrs (_: lib.mkOptionDefault); +} diff --git a/modules/README.md b/modules/README.md index 0d3e9233..ab9b3cbd 100644 --- a/modules/README.md +++ b/modules/README.md @@ -27,6 +27,7 @@ modules/ │ └── users.nix # macOS user management └── nixos/ # NixOS-specific modules ├── base/ # Base NixOS configuration + │ ├── auditd.nix # Audit trail for root sessions and credentials │ ├── btop.nix # System resource monitor │ ├── btrbk.nix # Local btrfs snapshots │ ├── core.nix # Core system settings diff --git a/modules/nixos/base/auditd.nix b/modules/nixos/base/auditd.nix new file mode 100644 index 00000000..45ce14e5 --- /dev/null +++ b/modules/nixos/base/auditd.nix @@ -0,0 +1,128 @@ +{ + config, + lib, + options, + pkgs, + ... +}: +let + cfg = config.modules.security.audit; + inherit (config.services) openssh; + + # Audit trail for login sessions, so that a stolen deployment key leaves a + # record of what each session did; see SECURITY.md ("Audit trail"). + # `auditctl -R` aborts at the first watch whose path is missing, leaving + # auditing off, so every watched path must exist when the rules load. + + # Secrets are symlinks into a per-deploy generation under the agenix ramfs, + # which a per-file watch would lose on the next deploy; watch the ramfs tree. + agenixSecrets = lib.optionals (options ? age) (lib.attrValues config.age.secrets); + hasAgenix = agenixSecrets != [ ]; + credentialDirs = lib.optional hasAgenix config.age.secretsMountPoint; + credentialFiles = lib.unique ( + map (secret: secret.path) (lib.filter (secret: !secret.symlink) agenixSecrets) + # The host key is also agenix's identity: reading it decrypts every secret. + ++ lib.optionals openssh.enable (map (key: key.path) openssh.hostKeys) + ++ lib.optionals hasAgenix config.age.identityPaths + ); + + identityFiles = map (name: "/etc/${name}") [ + "passwd" + "shadow" + "group" + "gshadow" + "sudoers" + ]; + + watchedDirs = credentialDirs ++ [ + "/etc/ssh" + "/etc/pam.d" + "/etc/nix" + "/etc/audit" + "/var/log/audit" + ]; + watchedFiles = credentialFiles ++ identityFiles ++ [ "/nix/var/nix/profiles/system" ]; + + watchFile = + key: perm: path: + "-a always,exit -F path=${path} -F perm=${perm} -k ${key}"; + watchDir = + key: perm: dir: + "-a always,exit -F dir=${dir} -F perm=${perm} -k ${key}"; + # A b64-only syscall rule misses 32-bit binaries. + arches = [ "b64" ] ++ lib.optional pkgs.stdenv.hostPlatform.isx86_64 "b32"; + perArch = rule: map (arch: "-a always,exit -F arch=${arch} ${rule}") arches; +in +{ + options.modules.security.audit.enable = lib.mkEnableOption '' + the Linux audit trail (auditd) for login sessions: every command run as + root, access to decrypted secrets and SSH host keys, and changes to SSH, + account, PAM, Nix trust and audit configuration. Logs stay in + /var/log/audit on the host + ''; + + config = lib.mkIf cfg.enable { + assertions = [ + { + # auditctl -R splits fields on whitespace; it does not shell-unquote paths. + assertion = lib.all (path: builtins.match ".*[[:space:]].*" path == null) ( + watchedDirs ++ watchedFiles + ); + message = "modules.security.audit: audited paths cannot contain whitespace in the native auditctl rule file."; + } + ]; + + security.auditd.enable = true; + security.audit = { + enable = true; + backlogLimit = lib.mkDefault 8192; + # The first matching exit rule sets the key: specific watches go first. + rules = lib.mkBefore ( + # Evaluated at fork: tasks without a login uid (everything systemd + # starts) get no audit context, so no per-syscall cost. + [ "-a never,task -F auid=unset" ] + ++ map (watchFile "credential-access" "rwa") credentialFiles + ++ map (watchDir "credential-access" "rwa") credentialDirs + ++ [ (watchDir "ssh-config" "wa" "/etc/ssh") ] + ++ map (watchFile "identity" "wa") identityFiles + ++ [ + (watchDir "identity" "wa" "/etc/pam.d") + (watchDir "nix-trust" "wa" "/etc/nix") + (watchFile "nix-trust" "wa" "/nix/var/nix/profiles/system") + (watchDir "audit-config" "wa" "/etc/audit") + (watchDir "audit-config" "wa" "/var/log/audit") + ] + ++ perArch "-S init_module,finit_module,delete_module -k kernel-module" + ++ perArch "-S execve,execveat -F euid=0 -F auid!=unset -k root-session" + ++ [ + "-a always,exit -S all -F exit=-EACCES -F auid!=unset -k access-denied" + "-a always,exit -S all -F exit=-EPERM -F auid!=unset -k access-denied" + ] + ); + }; + + security.auditd.settings = { + log_file = "/var/log/audit/audit.log"; + max_log_file_action = "rotate"; + max_log_file = 50; # MiB + num_logs = 5; + }; + + # Parents of watches that may not exist yet on a fresh system. + systemd.tmpfiles.rules = [ + "d /var/log/audit 0700 root root -" + "d /nix/var/nix/profiles 0755 root root -" + ]; + + # A watch binds to the inode present at load time, so load after the + # bind mounts (preservation), tmpfiles and the agenix ramfs. + systemd.services.audit-rules-nixos = { + after = [ + "local-fs.target" + "systemd-tmpfiles-setup.service" + ] + ++ lib.optional hasAgenix "agenix-install-secrets.service"; + unitConfig.RequiresMountsFor = lib.unique (watchedDirs ++ map dirOf watchedFiles); + }; + }; +} diff --git a/modules/nixos/base/ssh.nix b/modules/nixos/base/ssh.nix index 28d77a6a..656d5e37 100644 --- a/modules/nixos/base/ssh.nix +++ b/modules/nixos/base/ssh.nix @@ -14,8 +14,12 @@ # root login stays available for Colmena/MicroVM deployment, keys only PermitRootLogin = lib.mkDefault "prohibit-password"; PasswordAuthentication = false; # disable password login + # nixpkgs defaults it to true, and PAM then still accepts the Unix password + KbdInteractiveAuthentication = false; }; openFirewall = true; + # Only keys declared in Nix log in; a hand- or intruder-added key is ignored + authorizedKeysInHomedir = false; }; # NixOS appends an `Include` of systemd's ssh_config.d drop-in to diff --git a/outputs/aarch64-linux/tests/security-audit/expected.nix b/outputs/aarch64-linux/tests/security-audit/expected.nix new file mode 100644 index 00000000..bda15660 --- /dev/null +++ b/outputs/aarch64-linux/tests/security-audit/expected.nix @@ -0,0 +1,2 @@ +{ lib, outputs }: +import ../../../x86_64-linux/tests/security-audit/expected.nix { inherit lib outputs; } diff --git a/outputs/aarch64-linux/tests/security-audit/expr.nix b/outputs/aarch64-linux/tests/security-audit/expr.nix new file mode 100644 index 00000000..cdee96f2 --- /dev/null +++ b/outputs/aarch64-linux/tests/security-audit/expr.nix @@ -0,0 +1,2 @@ +{ lib, outputs }: +import ../../../x86_64-linux/tests/security-audit/expr.nix { inherit lib outputs; } diff --git a/outputs/aarch64-linux/tests/security-ssh-auth/expected.nix b/outputs/aarch64-linux/tests/security-ssh-auth/expected.nix new file mode 100644 index 00000000..cece43b6 --- /dev/null +++ b/outputs/aarch64-linux/tests/security-ssh-auth/expected.nix @@ -0,0 +1,25 @@ +{ + lib, + outputs, + myvars, +}: +lib.genAttrs (builtins.attrNames outputs.nixosConfigurations) ( + name: + { + # Neither password nor PAM keyboard-interactive login is offered. + PasswordAuthentication = false; + KbdInteractiveAuthentication = false; + # Undeclared keys in ~/.ssh/authorized_keys are not honoured. + authorizedKeysInHomedir = false; + # Only the interactive admin keys open a root shell; remote-builder host keys + # are pinned to the nix-daemon protocol, served by the system daemon. + unrestrictedRootKeys = builtins.length ( + myvars.mainSshAuthorizedKeys ++ myvars.secondaryAuthorizedKeys + ); + } + // lib.optionalAttrs (name == "aquamarine") { + builderKeys = map ( + key: ''restrict,command="nix-daemon --stdio --store daemon" ${key}'' + ) myvars.remoteBuilderClientKeys; + } +) diff --git a/outputs/aarch64-linux/tests/security-ssh-auth/expr.nix b/outputs/aarch64-linux/tests/security-ssh-auth/expr.nix new file mode 100644 index 00000000..591d56d7 --- /dev/null +++ b/outputs/aarch64-linux/tests/security-ssh-auth/expr.nix @@ -0,0 +1,21 @@ +{ + lib, + outputs, +}: +lib.genAttrs (builtins.attrNames outputs.nixosConfigurations) ( + name: + let + inherit (outputs.nixosConfigurations.${name}.config) services users; + in + { + inherit (services.openssh.settings) PasswordAuthentication KbdInteractiveAuthentication; + inherit (services.openssh) authorizedKeysInHomedir; + # Keys that may open a root session without a forced command. + unrestrictedRootKeys = builtins.length ( + lib.filter (key: !lib.hasPrefix "restrict," key) users.users.root.openssh.authorizedKeys.keys + ); + } + // lib.optionalAttrs (name == "aquamarine") { + builderKeys = lib.filter (lib.hasPrefix "restrict,") users.users.root.openssh.authorizedKeys.keys; + } +) diff --git a/outputs/default.nix b/outputs/default.nix index b390ed0f..79f04f83 100644 --- a/outputs/default.nix +++ b/outputs/default.nix @@ -214,6 +214,12 @@ in pkgs = nixpkgs.legacyPackages.${system}; networking = myvars.networking; }; + # Load the audit rules in a VM: one rule that fails to load leaves + # auditing off, which evaluation cannot catch. + security-auditd = import ../tests/security-auditd.nix { + pkgs = nixpkgs.legacyPackages.${system}; + inherit (inputs) agenix; + }; } ); diff --git a/outputs/x86_64-linux/tests/security-audit/expected.nix b/outputs/x86_64-linux/tests/security-audit/expected.nix new file mode 100644 index 00000000..adccbc45 --- /dev/null +++ b/outputs/x86_64-linux/tests/security-audit/expected.nix @@ -0,0 +1,24 @@ +{ + lib, + outputs, +}: +lib.genAttrs (builtins.attrNames outputs.nixosConfigurations) ( + name: + let + # Homelab servers reachable with the root deployment keys; not desktops. + audited = builtins.elem name [ + "youko" + "shoryu" + "shushou" + "aquamarine" + ]; + in + { + auditd = audited; + rootSession = audited; + sessionsOnly = audited; + # Only emitted where agenix has secrets: a watch on a missing path + # aborts loading the remaining rules. + agenixWatch = name == "youko"; + } +) diff --git a/outputs/x86_64-linux/tests/security-audit/expr.nix b/outputs/x86_64-linux/tests/security-audit/expr.nix new file mode 100644 index 00000000..972b134d --- /dev/null +++ b/outputs/x86_64-linux/tests/security-audit/expr.nix @@ -0,0 +1,16 @@ +{ + lib, + outputs, +}: +lib.genAttrs (builtins.attrNames outputs.nixosConfigurations) ( + name: + let + inherit (outputs.nixosConfigurations.${name}) config; + in + { + auditd = config.security.auditd.enable; + rootSession = lib.any (lib.hasSuffix "-k root-session") config.security.audit.rules; + sessionsOnly = lib.elem "-a never,task -F auid=unset" config.security.audit.rules; + agenixWatch = lib.elem "-a always,exit -F dir=/run/agenix.d -F perm=rwa -k credential-access" config.security.audit.rules; + } +) diff --git a/outputs/x86_64-linux/tests/security-service-hardening/expected.nix b/outputs/x86_64-linux/tests/security-service-hardening/expected.nix new file mode 100644 index 00000000..246fc288 --- /dev/null +++ b/outputs/x86_64-linux/tests/security-service-hardening/expected.nix @@ -0,0 +1,24 @@ +let + sandbox = { + CapabilityBoundingSet = ""; + NoNewPrivileges = true; + PrivateDevices = true; + ProtectSystem = "strict"; + SystemCallFilter = [ + "@system-service" + "~@clock @cpu-emulation @debug @module @mount @obsolete @privileged @raw-io @reboot @swap" + ]; + }; +in +{ + # Only 80/443. + caddy = sandbox // { + CapabilityBoundingSet = [ "CAP_NET_BIND_SERVICE" ]; + }; + rustfs = sandbox; + vmalert-homelab = sandbox; + # The NPU is allowed through DeviceAllow instead. + whisper-asr = sandbox // { + PrivateDevices = false; + }; +} diff --git a/outputs/x86_64-linux/tests/security-service-hardening/expr.nix b/outputs/x86_64-linux/tests/security-service-hardening/expr.nix new file mode 100644 index 00000000..cf7acb9b --- /dev/null +++ b/outputs/x86_64-linux/tests/security-service-hardening/expr.nix @@ -0,0 +1,21 @@ +{ lib, outputs }: +let + keys = [ + "CapabilityBoundingSet" + "NoNewPrivileges" + "PrivateDevices" + "ProtectSystem" + "SystemCallFilter" + ]; + units = + host: names: + lib.genAttrs names ( + n: lib.getAttrs keys outputs.nixosConfigurations.${host}.config.systemd.services.${n}.serviceConfig + ); +in +units "youko" [ + "caddy" + "rustfs" + "vmalert-homelab" +] +// units "aquamarine" [ "whisper-asr" ] diff --git a/outputs/x86_64-linux/tests/security-ssh-auth/expected.nix b/outputs/x86_64-linux/tests/security-ssh-auth/expected.nix new file mode 100644 index 00000000..cece43b6 --- /dev/null +++ b/outputs/x86_64-linux/tests/security-ssh-auth/expected.nix @@ -0,0 +1,25 @@ +{ + lib, + outputs, + myvars, +}: +lib.genAttrs (builtins.attrNames outputs.nixosConfigurations) ( + name: + { + # Neither password nor PAM keyboard-interactive login is offered. + PasswordAuthentication = false; + KbdInteractiveAuthentication = false; + # Undeclared keys in ~/.ssh/authorized_keys are not honoured. + authorizedKeysInHomedir = false; + # Only the interactive admin keys open a root shell; remote-builder host keys + # are pinned to the nix-daemon protocol, served by the system daemon. + unrestrictedRootKeys = builtins.length ( + myvars.mainSshAuthorizedKeys ++ myvars.secondaryAuthorizedKeys + ); + } + // lib.optionalAttrs (name == "aquamarine") { + builderKeys = map ( + key: ''restrict,command="nix-daemon --stdio --store daemon" ${key}'' + ) myvars.remoteBuilderClientKeys; + } +) diff --git a/outputs/x86_64-linux/tests/security-ssh-auth/expr.nix b/outputs/x86_64-linux/tests/security-ssh-auth/expr.nix new file mode 100644 index 00000000..591d56d7 --- /dev/null +++ b/outputs/x86_64-linux/tests/security-ssh-auth/expr.nix @@ -0,0 +1,21 @@ +{ + lib, + outputs, +}: +lib.genAttrs (builtins.attrNames outputs.nixosConfigurations) ( + name: + let + inherit (outputs.nixosConfigurations.${name}.config) services users; + in + { + inherit (services.openssh.settings) PasswordAuthentication KbdInteractiveAuthentication; + inherit (services.openssh) authorizedKeysInHomedir; + # Keys that may open a root session without a forced command. + unrestrictedRootKeys = builtins.length ( + lib.filter (key: !lib.hasPrefix "restrict," key) users.users.root.openssh.authorizedKeys.keys + ); + } + // lib.optionalAttrs (name == "aquamarine") { + builderKeys = lib.filter (lib.hasPrefix "restrict,") users.users.root.openssh.authorizedKeys.keys; + } +) diff --git a/tests/security-auditd.nix b/tests/security-auditd.nix new file mode 100644 index 00000000..5b0f116c --- /dev/null +++ b/tests/security-auditd.nix @@ -0,0 +1,100 @@ +{ pkgs, agenix }: +let + # Throwaway age identity and secret, generated at build time for the VM. + testSecrets = pkgs.runCommand "auditd-test-secrets" { nativeBuildInputs = [ pkgs.age ]; } '' + mkdir $out + age-keygen -o $out/key.txt + echo not-a-real-secret | age -r "$(age-keygen -y $out/key.txt)" -o $out/secret.age + ''; +in +pkgs.testers.runNixOSTest { + name = "security-auditd"; + nodes.machine = + { config, ... }: + { + imports = [ + ../modules/nixos/base/auditd.nix + agenix.nixosModules.default + ]; + modules.security.audit.enable = true; + services.openssh.enable = true; + age.identityPaths = [ "${testSecrets}/key.txt" ]; + age.secrets.test-secret.file = "${testSecrets}/secret.age"; + users.users.alice.isNormalUser = true; + environment.etc."audit-rule-count".text = toString (builtins.length config.security.audit.rules); + virtualisation.memorySize = 1024; + }; + testScript = '' + import shlex + + def ssh(cmd): + return machine.execute( + "ssh -i /root/id -o StrictHostKeyChecking=no root@localhost " + shlex.quote(cmd) + ) + + # auditd writes asynchronously, so poll; `-ul 0` keeps records whose + # login uid (auid) is root, i.e. from the SSH session. Without + # --input-logs, ausearch reads stdin when it is not a terminal. + def wait_logged(key, match, flags="", count=1): + machine.wait_until_succeeds( + f"test $(ausearch --input-logs -i -k {key} {flags} | grep -c -- {shlex.quote(match)}) -ge {count}" + ) + + machine.wait_for_unit("audit-rules-nixos.service") + machine.wait_for_unit("auditd.service") + machine.wait_for_unit("sshd.service") + + with subtest("every rule loads and auditing is enabled"): + t.assertIn("enabled 1", machine.succeed("auditctl -s")) + rules = machine.succeed("auditctl -l") + t.assertEqual(len(rules.splitlines()), int(machine.succeed("cat /etc/audit-rule-count"))) + t.assertIn("-w /run/agenix.d -p rwa -k credential-access", rules) + t.assertIn("-a never,task -F auid=-1", rules) + + # The test driver's shell is a service: this setup is not audited. + machine.succeed( + "ssh-keygen -q -t ed25519 -N ''' -f /root/id", + "install -d -m 700 /root/.ssh", + "cat /root/id.pub >> /root/.ssh/authorized_keys", + ) + + with subtest("an SSH root session logs its commands and secret reads"): + ssh("cat /run/agenix/test-secret; ${pkgs.hello}/bin/hello") + wait_logged("root-session", "hello", "-ul 0") + # sshd is built without libaudit; PAM records the login and its address. + machine.succeed("ausearch --input-logs -i -m USER_START -ul 0 | grep -q 'addr=::1 terminal=ssh'") + wait_logged("credential-access", "name=/run/agenix/test-secret", "-ul 0") + + with subtest("a service is not audited; the same actions from a session are"): + machine.succeed( + "systemd-run --wait -p Environment=PATH=${pkgs.coreutils}/bin ${pkgs.runtimeShell} -c " + "'cat /run/agenix/test-secret >/dev/null; touch /etc/ssh/from-service'" + ) + ssh("cat /run/agenix/test-secret >/dev/null; touch /etc/ssh/from-session") + wait_logged("ssh-config", "from-session", "-ul 0") + wait_logged("credential-access", "name=/run/agenix/test-secret", "-ul 0", count=2) + machine.succeed("test -e /etc/ssh/from-service") + machine.fail("ausearch --input-logs -i -k ssh-config | grep -q from-service") + # auditctl's own rule loads also carry the key (CONFIG_CHANGE, no auid). + machine.fail("ausearch --input-logs -i -k credential-access | grep auid=unset | grep -q comm=cat") + machine.fail("ausearch --input-logs -i -k root-session | grep -q from-service") + # systemd-executor opens the agenix mount point with O_PATH while it + # builds a sandboxed unit's namespace; it has no auid either. + machine.succeed("systemctl restart nscd") + machine.fail("ausearch --input-logs -i -k credential-access | grep -q comm=.nsncd") + + with subtest("the agenix watch survives a new secrets generation"): + machine.succeed("/run/current-system/activate") + machine.succeed("readlink /run/agenix | grep -q '/2$'") + ssh("cat /run/agenix/test-secret >/dev/null") + wait_logged("credential-access", "name=/run/agenix/test-secret", "-ul 0", count=3) + + with subtest("denials in a login session are recorded"): + ssh("su alice -s /bin/sh -c 'cat /etc/shadow'") + wait_logged("access-denied", "shadow", "-ul 0") + + with subtest("auditd keeps its log; the journal keeps a copy"): + machine.succeed("test -s /var/log/audit/audit.log") + machine.succeed("journalctl _TRANSPORT=audit -q | grep -q root-session") + ''; +}