Merge pull request #272 from ryan4yin/fix/system-audit-bugs

fix: resolve audited system component bugs

hosts/idols-ai/disko-fs.nix

@@ -87,13 +87,6 @@
                       "noatime"
                     ];
                   };
-                  "@guix" = {
-                    mountpoint = "/gnu";
-                    mountOptions = [
-                      "compress-force=zstd:1"
-                      "noatime"
-                    ];
-                  };
                   "@persistent" = {
                     mountpoint = "/persistent";
                     mountOptions = [

hosts/idols-ai/hardware-configuration.nix

@@ -74,7 +74,7 @@
     "exfat"
   ];
 
-  # LUKS initrd, all fileSystems (/, /boot, /btr_pool, /nix, /gnu, /persistent, /snapshots, /tmp, /swap)
+  # LUKS initrd, all fileSystems (/, /boot, /btr_pool, /nix, /persistent, /snapshots, /tmp, /swap)
   # and swap (including /swap/swapfile bind and swapDevices) are managed by disko (disko-fs.nix).
 
   # Enables DHCP on each ethernet and wireless interface. In case of scripted networking

hosts/idols-aquamarine/grafana/datasources.nix

@@ -88,31 +88,37 @@
         };
         editable = false;
       }
-      {
-        # https://grafana.com/docs/grafana/latest/datasources/postgres/configure/
-        name = "postgres-playground";
-        type = "postgres";
-        url = "postgres.writefor.fun:5432";
-        user = "playground";
-        secureJsonData = {
-          password = "$__file{${config.age.secrets."grafana-admin-password".path}}";
-        };
-        jsonData = {
-          database = "playground";
-          sslmode = "verify-full"; # disable/require/verify-ca/verify-full
-          maxOpenConns = 50;
-          maxIdleConns = 250;
-          maxIdleConnsAuto = true;
-          connMaxLifetime = 14400;
-          timeInterval = "1m";
-          timescaledb = false;
-          postgresVersion = 1500; # 15.xx
-          # tls
-          tlsConfigurationMethod = "file-path";
-          sslRootCertFile = ../../../certs/ecc-ca.crt;
-        };
-        editable = false;
-      }
+      # Disabled: the `playground` role has no password and remote logins
+      # require scram-sha-256, so this datasource could never authenticate.
+      # Re-enable once a `playground` DB password is provisioned
+      # (docs/system-audit-2026-09-19.md, item 1.5).
+      /*
+        {
+          # https://grafana.com/docs/grafana/latest/datasources/postgres/configure/
+          name = "postgres-playground";
+          type = "postgres";
+          url = "postgres.writefor.fun:5432";
+          user = "playground";
+          secureJsonData = {
+            password = "$__file{${config.age.secrets."postgres-playground-password".path}}";
+          };
+          jsonData = {
+            database = "playground";
+            sslmode = "verify-full"; # disable/require/verify-ca/verify-full
+            maxOpenConns = 50;
+            maxIdleConns = 250;
+            maxIdleConnsAuto = true;
+            connMaxLifetime = 14400;
+            timeInterval = "1m";
+            timescaledb = false;
+            postgresVersion = 1500; # 15.xx
+            # tls
+            tlsConfigurationMethod = "file-path";
+            sslRootCertFile = ../../../certs/ecc-ca.crt;
+          };
+          editable = false;
+        }
+      */
       {
         name = "infinity-dataviewer";
         type = "yesoreyeram-infinity-datasource";

hosts/idols-aquamarine/grafana/default.nix

@@ -20,7 +20,7 @@
         http_addr = "127.0.0.1";
         http_port = 3351;
         protocol = "http";
-        domain = "grafana.writefo.fun";
+        domain = "grafana.writefor.fun";
         # Redirect to correct domain if the host header does not match the domain. Prevents DNS rebinding attacks.
         serve_from_sub_path = false;
         # Add subpath to the root_url if serve_from_sub_path is true

hosts/idols-aquamarine/monitoring/README.md

@@ -6,3 +6,16 @@
   - Collection of Prometheus alerting rules.
 - [victoria-metrics-k8s-stack/files/rules](https://github.com/VictoriaMetrics/helm-charts/tree/master/charts/victoria-metrics-k8s-stack/files/rules/generated)
   - Alert Rules & Recoding Rules used by kube-prometheus-stack.
+
+## TODO
+
+- **Dead-man switch**: `Watchdog` in `alert_rules/general.yml` always fires (`vector(1)`) but is
+  routed to the `null` receiver in `alert.nix`, so a dead alerting pipeline looks identical to a
+  healthy one. Route it to an external heartbeat service (healthchecks.io, Uptime Kuma, PagerDuty
+  DeadMansSnitch) that notifies when the heartbeat stops, or run an out-of-band probe against
+  Alertmanager.
+- **Backup freshness**: nothing alerts on "no recent successful backup". The systemd `failed` state
+  only catches crashes, not a job that never ran or a backup that is silently stale. Export the last
+  successful backup timestamp (restic prometheus exporter; btrfs snapshot time via the node-exporter
+  textfile collector) and alert when it exceeds the SLO, and add an off-host btrbk `target` so local
+  snapshots are not the only copy.

hosts/idols-aquamarine/monitoring/alert.nix

@@ -20,7 +20,9 @@
       # Path to the files with alerting and/or recording rules.
       rule = [
         "${./alert_rules}/*.yml"
+        "${./alert_rules}/*.yaml"
         "${./recoding_rules}/*.yml"
+        "${./recoding_rules}/*.yaml"
       ];
       # https://docs.victoriametrics.com/victoriametrics/vmalert/#link-to-alert-source
       # Set this two args to generate the correct `.GeneratorURL`

hosts/idols-aquamarine/restic.nix

@@ -5,77 +5,83 @@ let
   rcloneConfigFile = "/etc/agenix/rclone-conf-for-restic-backup";
 in
 {
+  # DISABLED: this backup has never worked. It needs three agenix secrets that
+  # do not exist in nix-secrets yet (`restic-password`, `ssh-key-for-restic-backup`,
+  # and an rclone config with the `smb-downloads` remote; the existing
+  # `rclone.conf` secret is only defined for the desktop secret set). Re-enable
+  # once the secrets are provisioned.
+  #
   # https://github.com/NixOS/nixpkgs/blob/nixos-26.05/nixos/modules/services/backup/restic.nix
-  services.restic.backups = {
-    homelab-backup = {
-      inherit passwordFile;
-      initialize = true; # Initialize the repository if it doesn't exist.
-      repository = "rclone:smb-downloads:/Downloads/kubevirt-backup/"; # backup to a rclone remote
+  /*
+    services.restic.backups = {
+      homelab-backup = {
+        inherit passwordFile;
+        initialize = true; # Initialize the repository if it doesn't exist.
+        repository = "rclone:smb-downloads:/Downloads/kubevirt-backup/"; # backup to a rclone remote
 
-      # rclone related
-      # rcloneOptions = {
-      #   bwlimit = "100M";  # Limit the bandwidth used by rclone.
-      # };
-      inherit rcloneConfigFile;
+        # rclone related
+        # rcloneOptions = {
+        #   bwlimit = "100M";  # Limit the bandwidth used by rclone.
+        # };
+        inherit rcloneConfigFile;
 
-      # Which local paths to backup, in addition to ones specified via `dynamicFilesFrom`.
-      paths = [
-        "/tmp/restic-backup-temp"
-      ];
-      #
-      # A script that produces a list of files to back up.  The
-      # results of this command are given to the '--files-from'
-      # option. The result is merged with paths specified via `paths`.
-      # dynamicFilesFrom = "find /home/matt/git -type d -name .git";
-      #
-      # Patterns to exclude when backing up. See
-      #   https://restic.readthedocs.io/en/latest/040_backup.html#excluding-files
-      # for details on syntax.
-      exclude = [ ];
+        # Which local paths to backup, in addition to ones specified via `dynamicFilesFrom`.
+        paths = [
+          "/tmp/restic-backup-temp"
+        ];
+        #
+        # A script that produces a list of files to back up.  The
+        # results of this command are given to the '--files-from'
+        # option. The result is merged with paths specified via `paths`.
+        # dynamicFilesFrom = "find /home/matt/git -type d -name .git";
+        #
+        # Patterns to exclude when backing up. See
+        #   https://restic.readthedocs.io/en/latest/040_backup.html#excluding-files
+        # for details on syntax.
+        exclude = [ ];
 
-      # A script that must run before starting the backup process.
-      backupPrepareCommand = ''
-        ${pkgs.nushell}/bin/nu -c '
-          let kubevirt_nodes = [
-            "kubevirt-shoryu"
-            "kubevirt-shushou"
-            "kubevirt-youko"
-          ]
+        # A script that must run before starting the backup process.
+        backupPrepareCommand = ''
+          ${pkgs.nushell}/bin/nu -c '
+            let kubevirt_nodes = [
+              "kubevirt-shoryu"
+              "kubevirt-shushou"
+              "kubevirt-youko"
+            ]
 
-          kubevirt_nodes | each {|it|
-            rsync -avz \
-            -e "ssh -i ${sshKeyPath}"  \
-            $"($it):/perissitent/" $"/tmp/restic-backup-temp/($it)"
-          }
-        '
-      '';
-      # A script that must run after finishing the backup process.
-      backupCleanupCommand = "rm -rf /tmp/restic-backup-temp";
+            kubevirt_nodes | each {|it|
+              ^${pkgs.rsync}/bin/rsync -avz -e $"ssh -i ${sshKeyPath}" $"($it):/persistent/" $"/tmp/restic-backup-temp/($it)"
+            }
+          '
+        '';
+        # A script that must run after finishing the backup process.
+        backupCleanupCommand = "rm -rf /tmp/restic-backup-temp";
 
-      # Extra extended options to be passed to the restic --option flag.
-      # extraOptions = [];
+        # Extra extended options to be passed to the restic --option flag.
+        # extraOptions = [];
 
-      # Extra arguments passed to restic backup.
-      # extraBackupArgs = [
-      #   "--exclude-file=/etc/restic/excludes-list"
-      # ];
+        # Extra arguments passed to restic backup.
+        # extraBackupArgs = [
+        #   "--exclude-file=/etc/restic/excludes-list"
+        # ];
 
-      # repository = "/mnt/backup-hdd"; # backup to a local directory
-      # When to run the backup. See {manpage}`systemd.timer(5)` for details.
-      timerConfig = {
-        OnCalendar = "01:30";
-        RandomizedDelaySec = "1h";
+        # repository = "/mnt/backup-hdd"; # backup to a local directory
+        # When to run the backup. See {manpage}`systemd.timer(5)` for details.
+        timerConfig = {
+          OnCalendar = "01:30";
+          RandomizedDelaySec = "1h";
+        };
+        # A list of options (--keep-* et al.) for 'restic forget --prune',
+        # to automatically prune old snapshots.
+        # The 'forget' command is run *after* the 'backup' command, so
+        # keep that in mind when constructing the --keep-* options.
+        pruneOpts = [
+          "--keep-daily 3"
+          "--keep-weekly 3"
+          "--keep-monthly 3"
+          "--keep-yearly 3"
+        ];
       };
-      # A list of options (--keep-* et al.) for 'restic forget --prune',
-      # to automatically prune old snapshots.
-      # The 'forget' command is run *after* the 'backup' command, so
-      # keep that in mind when constructing the --keep-* options.
-      pruneOpts = [
-        "--keep-daily 3"
-        "--keep-weekly 3"
-        "--keep-monthly 3"
-        "--keep-yearly 3"
-      ];
     };
-  };
+  */
 }

modules/nixos/base/trash.nix

@@ -56,6 +56,8 @@ in
     timerConfig = {
       OnCalendar = "daily";
       RandomizedDelaySec = "15min";
+      # run the skipped purge on the next boot if the host was off/suspended.
+      Persistent = true;
     };
   };
 }

modules/nixos/desktop/guix.nix

@@ -1,21 +0,0 @@
-{
-  # https://github.com/NixOS/nixpkgs/blob/master/nixos/modules/services/misc/guix/default.nix
-  services.guix = {
-    enable = true;
-    # The store directory where the Guix service will serve to/from.
-    # NOTE: most of the cached builds are assumed to be in `/gnu/store`.
-    storeDir = "/gnu/store";
-    # The state directory where Guix service will store its data such as its
-    # user-specific profiles, cache, and state files.
-    stateDir = "/var";
-    gc = {
-      enable = true;
-      # https://guix.gnu.org/en/manual/en/html_node/Invoking-guix-gc.html
-      extraArgs = [
-        "--delete-generations=1m"
-        "--free-space=10G"
-        "--optimize"
-      ];
-    };
-  };
-}

outputs/default.nix

@@ -154,8 +154,19 @@ in
   evalTests = lib.lists.all (it: it.evalTests == { }) allSystemValues;
 
   checks = forAllSystems (system: {
-    # eval-tests per system
-    eval-tests = allSystems.${system}.evalTests == { };
+    # eval-tests per system. `nix flake check` requires every check to be a
+    # derivation, so wrap the boolean result in one instead of returning a bool.
+    eval-tests =
+      let
+        pkgs = nixpkgs.legacyPackages.${system};
+        results = allSystems.${system}.evalTests;
+      in
+      pkgs.runCommand "eval-tests" { } (
+        if results == { } then
+          "touch $out"
+        else
+          "echo 'eval tests failed: evalTests is not empty' >&2; exit 1"
+      );
 
     pre-commit-check = pre-commit-hooks.lib.${system}.run {
       src = mylib.relativeToRoot ".";