security: hardening pass 1 (AppArmor, firewall default, loopback metrics, k3s perms, SSH X11) (#266)

* security: make firewall secure-by-default, disable explicitly on servers

Flip the base firewall default to ON so new hosts are protected by default. Servers keep the firewall off (trusted internal LAN, WAN protected at the router) via explicit overrides in modules/nixos/server/{server,server-aarch64}.nix. Behavior-preserving for all 18 current hosts; adds a security-firewall eval test guarding the per-host state.

* test: fix security-firewall eval test to scalar per-host values

* security: enable AppArmor (complain) on all Linux hosts

* security: bind aquamarine metrics exporters to loopback

* security: restrict k3s kubeconfig file mode to 600

* security: disable SSH X11 forwarding on servers, keep on desktops

* security: add conservative systemd hardening to aquamarine services

* docs(apparmor): correct FHS-alias comment (nixpkgs has no broad FHS layer)

nixpkgs only creates /run/current-system, /usr/bin/env and /bin/sh; it does not
provide a broad FHS->store alias tree, so FHS-oriented abstractions are incomplete
on NixOS. They are safe only because the profiles run in complain mode.

* fix(aquamarine): scrape same-host exporters over loopback

The v2ray/postgres/sftpgo exporters were bound to 127.0.0.1 but VictoriaMetrics
scraped them via the host's routable IP, so those scrapes refused connections.
Point the three same-host scrape targets at 127.0.0.1 (VM runs on the same host).
node-exporter keeps the host IP since it binds 0.0.0.0.

* Revert "security: add conservative systemd hardening to aquamarine services"

This reverts commit f9cf99dadb.

hardening/apparmor/default.nix

@@ -4,56 +4,53 @@
   ...
 }:
 {
+  # AppArmor: activate the LSM + load policies. Roll out in complain mode first
+  # (log violations, never block) so nothing can break; promote profiles to
+  # "enforce" individually once stable.
   services.dbus.apparmor = "enabled";
+
   security.apparmor = {
     enable = true;
 
-    # kill process that are not confined but have apparmor profiles enabled
+    # Do not SIGTERM running unconfined-but-confinable processes yet.
-    killUnconfinedConfinables = true;
+    # Safe to flip to true later now that no global default-deny profile is active.
-    packages = with pkgs; [
+    killUnconfinedConfinables = false;
-      apparmor-utils
+
-      apparmor-profiles
+    # Packages contributing to AppArmor's include path (abstractions).
-    ];
+    # NOTE: these abstractions are FHS-oriented and reference FHS paths. nixpkgs does NOT
+    # provide a broad FHS->store alias layer (only /run/current-system, /usr/bin/env and
+    # /bin/sh are created), so some referenced paths do not resolve on NixOS and the
+    # resulting profiles are incomplete. They are safe only because they run in complain
+    # mode (log-only, no blocking); see hardening/README.md.
+    packages = [ pkgs.apparmor-profiles ];
 
-    # apparmor policies
     policies = {
+      # Global default-deny scaffold. DANGEROUS to enable: `/**` matches every binary and
+      # the empty block allows nothing. Keep disabled until per-app profiles exist.
       "default_deny" = {
-        enforce = false;
+        state = "disable";
-        enable = false;
+        profile = "profile default_deny /** { }";
-        profile = ''
-          profile default_deny /** { }
-        '';
       };
 
+      # Confine sudo; complain mode so a missing rule logs instead of blocking.
       "sudo" = {
-        enforce = false;
+        state = "complain";
-        enable = false;
         profile = ''
-          ${pkgs.sudo}/bin/sudo {
+          abi <abi/4.0>,
+          include <tunables/global>
+
+          profile ${pkgs.sudo}/bin/sudo {
+            include <abstractions/base>
             file /** rwlkUx,
           }
         '';
       };
 
+      # nix runs unconfined (no restriction); inert, kept disabled.
       "nix" = {
-        enforce = false;
+        state = "disable";
-        enable = false;
+        profile = "profile ${config.nix.package}/bin/nix { unconfined, }";
-        profile = ''
-          ${config.nix.package}/bin/nix {
-            unconfined,
-          }
-        '';
       };
     };
   };
-
-  environment.systemPackages = with pkgs; [
-    apparmor-bin-utils
-    apparmor-profiles
-    apparmor-parser
-    libapparmor
-    apparmor-kernel-patches
-    apparmor-pam
-    apparmor-utils
-  ];
 }

hosts/idols-aquamarine/monitoring/victoriametrics.nix

@@ -85,7 +85,8 @@ in
           metrics_path = "/metrics";
           static_configs = [
             {
-              targets = [ "${myvars.networking.hostsAddr.aquamarine.ipv4}:9153" ];
+              # same-host exporter bound to loopback (127.0.0.1:9153)
+              targets = [ "127.0.0.1:9153" ];
               labels.type = "app";
               labels.app = "v2ray";
               labels.host = "aquamarine";
@@ -116,7 +117,8 @@ in
           metrics_path = "/metrics";
           static_configs = [
             {
-              targets = [ "${myvars.networking.hostsAddr.aquamarine.ipv4}:9187" ];
+              # same-host exporter bound to loopback (127.0.0.1:9187)
+              targets = [ "127.0.0.1:9187" ];
               labels.type = "app";
               labels.app = "postgresql";
               labels.host = "aquamarine";
@@ -131,7 +133,8 @@ in
           metrics_path = "/metrics";
           static_configs = [
             {
-              targets = [ "${myvars.networking.hostsAddr.aquamarine.ipv4}:10000" ];
+              # same-host exporter bound to loopback (127.0.0.1:10000)
+              targets = [ "127.0.0.1:10000" ];
               labels.type = "app";
               labels.app = "sftpgo";
               labels.host = "aquamarine";

hosts/idols-aquamarine/postgresql.nix

@@ -121,7 +121,8 @@ in
 
   services.prometheus.exporters.postgres = {
     enable = true;
-    listenAddress = "0.0.0.0";
+    # loopback only: scraped by VictoriaMetrics on the same host
+    listenAddress = "127.0.0.1";
     port = 9187;
     user = "postgres-exporter";
     group = "postgres-exporter";

hosts/idols-aquamarine/proxy.nix

@@ -82,7 +82,8 @@
   # https://github.com/wi1dcard/v2ray-exporter
   services.prometheus.exporters.v2ray = {
     enable = true;
-    listenAddress = "0.0.0.0";
+    # loopback only: scraped by VictoriaMetrics on the same host
+    listenAddress = "127.0.0.1";
     port = 9153;
     openFirewall = false;
     v2rayEndpoint = "127.0.0.1:54321";

hosts/idols-aquamarine/sftpgo.nix

@@ -95,7 +95,8 @@ in
       # prometheus metrics
       telemetry = {
         bind_port = 10000;
-        bind_address = "0.0.0.0";
+        # loopback only: scraped by VictoriaMetrics on the same host
+        bind_address = "127.0.0.1";
         # auth_user_file = "";
       };
       # multi-factor authentication settings

lib/genK3sServerModule.nix

@@ -60,7 +60,7 @@ in
       let
         flagList = [
           "--write-kubeconfig=${kubeconfigFile}"
-          "--write-kubeconfig-mode=644"
+          "--write-kubeconfig-mode=600"
           "--service-node-port-range=80-32767"
           "--kube-apiserver-arg='--allow-privileged=true'" # required by kubevirt
           "--data-dir /var/lib/rancher/k3s"

modules/nixos/base/default.nix

@@ -1,4 +1,4 @@
 { mylib, ... }:
 {
-  imports = mylib.scanPaths ./.;
+  imports = mylib.scanPaths ./. ++ [ (mylib.relativeToRoot "hardening/apparmor") ];
 }

modules/nixos/base/ssh.nix

@@ -1,12 +1,15 @@
 { lib, ... }:
 {
-  # Or disable the firewall altogether.
+  # Secure by default: firewall ON everywhere unless a host explicitly disables it
-  networking.firewall.enable = lib.mkDefault false;
+  # (servers disable it in modules/nixos/server/{server,server-aarch64}.nix).
+  networking.firewall.enable = lib.mkDefault true;
   # Enable the OpenSSH daemon.
   services.openssh = {
     enable = true;
     settings = {
-      X11Forwarding = true;
+      # Secure by default: X11 forwarding off everywhere; desktops re-enable it
+      # in modules/nixos/desktop/ssh.nix (needed for GUI forwarding).
+      X11Forwarding = lib.mkDefault false;
       # root user is used for remote deployment, so we need to allow it
       PermitRootLogin = "prohibit-password";
       PasswordAuthentication = false; # disable password login

modules/nixos/desktop/ssh.nix

@@ -0,0 +1,5 @@
+{
+  # Desktops keep X11 forwarding (current behavior, needed for GUI forwarding);
+  # servers default to off (see modules/nixos/base/ssh.nix).
+  services.openssh.settings.X11Forwarding = true;
+}

modules/nixos/server/server-aarch64.nix

@@ -1,6 +1,7 @@
 {
   lib,
   pkgs,
+  mylib,
   ...
 }:
 {
@@ -15,8 +16,15 @@
     ../base/user-group.nix
 
     ../../base
+    # AppArmor is wired via modules/nixos/base/default.nix for other hosts; this
+    # aarch64 server imports base files individually, so add it explicitly.
+    (mylib.relativeToRoot "hardening/apparmor")
   ];
 
+  # Servers run on the trusted internal LAN (NAT'd; WAN protected at the router).
+  # Keep the firewall off here; the secure default is ON (see modules/nixos/base/ssh.nix).
+  networking.firewall.enable = false;
+
   # Fix: jasper is marked as broken, refusing to evaluate.
   environment.enableAllTerminfo = lib.mkForce false;
 }

modules/nixos/server/server.nix

@@ -4,4 +4,8 @@
     ../base
     ../../base
   ];
+
+  # Servers run on the trusted internal LAN (NAT'd; WAN protected at the router).
+  # Keep the firewall off here; the secure default is ON (see modules/nixos/base/ssh.nix).
+  networking.firewall.enable = false;
 }

outputs/aarch64-linux/tests/security-apparmor/expected.nix

@@ -0,0 +1,5 @@
+{
+  lib,
+  outputs,
+}:
+lib.genAttrs (builtins.attrNames outputs.nixosConfigurations) (_: true)

outputs/aarch64-linux/tests/security-apparmor/expr.nix

@@ -0,0 +1,7 @@
+{
+  lib,
+  outputs,
+}:
+lib.genAttrs (builtins.attrNames outputs.nixosConfigurations) (
+  name: outputs.nixosConfigurations.${name}.config.security.apparmor.enable
+)

outputs/aarch64-linux/tests/security-firewall/expected.nix

@@ -0,0 +1,7 @@
+{
+  lib,
+  outputs,
+}:
+lib.genAttrs (builtins.attrNames outputs.nixosConfigurations) (
+  name: name == "ai-niri" || name == "shoukei-niri"
+)

outputs/aarch64-linux/tests/security-firewall/expr.nix

@@ -0,0 +1,7 @@
+{
+  lib,
+  outputs,
+}:
+lib.genAttrs (builtins.attrNames outputs.nixosConfigurations) (
+  name: outputs.nixosConfigurations.${name}.config.networking.firewall.enable
+)

outputs/aarch64-linux/tests/security-k3s-kubeconfig/expected.nix

@@ -0,0 +1,10 @@
+{
+  lib,
+  outputs,
+}:
+lib.genAttrs (builtins.attrNames outputs.nixosConfigurations) (name: {
+  # Only the control-plane (master) nodes write the admin kubeconfig; agents
+  # (workers) connect to the master and have no --write-kubeconfig-* flags.
+  mode600 = lib.hasInfix "-master-" name;
+  mode644 = false;
+})

outputs/aarch64-linux/tests/security-k3s-kubeconfig/expr.nix

@@ -0,0 +1,15 @@
+{
+  lib,
+  outputs,
+}:
+lib.genAttrs (builtins.attrNames outputs.nixosConfigurations) (
+  name:
+  let
+    isK3s = lib.hasPrefix "k3s" name;
+    flags = if isK3s then outputs.nixosConfigurations.${name}.config.services.k3s.extraFlags else "";
+  in
+  {
+    mode600 = lib.hasInfix "--write-kubeconfig-mode=600" flags;
+    mode644 = lib.hasInfix "--write-kubeconfig-mode=644" flags;
+  }
+)

outputs/aarch64-linux/tests/security-ssh-x11/expected.nix

@@ -0,0 +1,7 @@
+{
+  lib,
+  outputs,
+}:
+lib.genAttrs (builtins.attrNames outputs.nixosConfigurations) (
+  name: name == "ai-niri" || name == "shoukei-niri"
+)

outputs/aarch64-linux/tests/security-ssh-x11/expr.nix

@@ -0,0 +1,7 @@
+{
+  lib,
+  outputs,
+}:
+lib.genAttrs (builtins.attrNames outputs.nixosConfigurations) (
+  name: outputs.nixosConfigurations.${name}.config.services.openssh.settings.X11Forwarding
+)

outputs/x86_64-linux/tests/security-apparmor/expected.nix

@@ -0,0 +1,5 @@
+{
+  lib,
+  outputs,
+}:
+lib.genAttrs (builtins.attrNames outputs.nixosConfigurations) (_: true)

outputs/x86_64-linux/tests/security-apparmor/expr.nix

@@ -0,0 +1,7 @@
+{
+  lib,
+  outputs,
+}:
+lib.genAttrs (builtins.attrNames outputs.nixosConfigurations) (
+  name: outputs.nixosConfigurations.${name}.config.security.apparmor.enable
+)

outputs/x86_64-linux/tests/security-aquamarine-metrics/expected.nix

@@ -0,0 +1,9 @@
+{
+  lib,
+  outputs,
+}:
+{
+  v2rayExporter = "127.0.0.1";
+  pgExporter = "127.0.0.1";
+  sftpgoTelemetry = "127.0.0.1";
+}

outputs/x86_64-linux/tests/security-aquamarine-metrics/expr.nix

@@ -0,0 +1,12 @@
+{
+  lib,
+  outputs,
+}:
+let
+  aqua = outputs.nixosConfigurations.aquamarine.config;
+in
+{
+  v2rayExporter = aqua.services.prometheus.exporters.v2ray.listenAddress;
+  pgExporter = aqua.services.prometheus.exporters.postgres.listenAddress;
+  sftpgoTelemetry = aqua.services.sftpgo.settings.telemetry.bind_address;
+}

outputs/x86_64-linux/tests/security-firewall/expected.nix

@@ -0,0 +1,7 @@
+{
+  lib,
+  outputs,
+}:
+lib.genAttrs (builtins.attrNames outputs.nixosConfigurations) (
+  name: name == "ai-niri" || name == "shoukei-niri"
+)

outputs/x86_64-linux/tests/security-firewall/expr.nix

@@ -0,0 +1,7 @@
+{
+  lib,
+  outputs,
+}:
+lib.genAttrs (builtins.attrNames outputs.nixosConfigurations) (
+  name: outputs.nixosConfigurations.${name}.config.networking.firewall.enable
+)

outputs/x86_64-linux/tests/security-k3s-kubeconfig/expected.nix

@@ -0,0 +1,10 @@
+{
+  lib,
+  outputs,
+}:
+lib.genAttrs (builtins.attrNames outputs.nixosConfigurations) (name: {
+  # Only the control-plane (master) nodes write the admin kubeconfig; agents
+  # (workers) connect to the master and have no --write-kubeconfig-* flags.
+  mode600 = lib.hasInfix "-master-" name;
+  mode644 = false;
+})

outputs/x86_64-linux/tests/security-k3s-kubeconfig/expr.nix

@@ -0,0 +1,15 @@
+{
+  lib,
+  outputs,
+}:
+lib.genAttrs (builtins.attrNames outputs.nixosConfigurations) (
+  name:
+  let
+    isK3s = lib.hasPrefix "k3s" name;
+    flags = if isK3s then outputs.nixosConfigurations.${name}.config.services.k3s.extraFlags else "";
+  in
+  {
+    mode600 = lib.hasInfix "--write-kubeconfig-mode=600" flags;
+    mode644 = lib.hasInfix "--write-kubeconfig-mode=644" flags;
+  }
+)

outputs/x86_64-linux/tests/security-ssh-x11/expected.nix

@@ -0,0 +1,7 @@
+{
+  lib,
+  outputs,
+}:
+lib.genAttrs (builtins.attrNames outputs.nixosConfigurations) (
+  name: name == "ai-niri" || name == "shoukei-niri"
+)

outputs/x86_64-linux/tests/security-ssh-x11/expr.nix

@@ -0,0 +1,7 @@
+{
+  lib,
+  outputs,
+}:
+lib.genAttrs (builtins.attrNames outputs.nixosConfigurations) (
+  name: outputs.nixosConfigurations.${name}.config.services.openssh.settings.X11Forwarding
+)