From ff961c1bc900afe832f1a4f98c1f410a09a3a2d0 Mon Sep 17 00:00:00 2001 From: Ryan Yin Date: Sat, 19 Sep 2026 12:11:03 +0800 Subject: [PATCH 1/6] fix(flake): make the eval-tests check a derivation checks..eval-tests returned a boolean, so nix flake check failed with "not a derivation". Wrap the evalTests result in a runCommand so flake check actually evaluates it. --- outputs/default.nix | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/outputs/default.nix b/outputs/default.nix index 6f8404c1..4b66723b 100644 --- a/outputs/default.nix +++ b/outputs/default.nix @@ -154,8 +154,19 @@ in evalTests = lib.lists.all (it: it.evalTests == { }) allSystemValues; checks = forAllSystems (system: { - # eval-tests per system - eval-tests = allSystems.${system}.evalTests == { }; + # eval-tests per system. `nix flake check` requires every check to be a + # derivation, so wrap the boolean result in one instead of returning a bool. + eval-tests = + let + pkgs = nixpkgs.legacyPackages.${system}; + results = allSystems.${system}.evalTests; + in + pkgs.runCommand "eval-tests" { } ( + if results == { } then + "touch $out" + else + "echo 'eval tests failed: evalTests is not empty' >&2; exit 1" + ); pre-commit-check = pre-commit-hooks.lib.${system}.run { src = mylib.relativeToRoot "."; From d78f6badd402ee99f56a080c6fc85f1e1dd32e07 Mon Sep 17 00:00:00 2001 From: Ryan Yin Date: Sat, 19 Sep 2026 12:11:07 +0800 Subject: [PATCH 2/6] fix(aquamarine): repair monitoring config Load both *.yml and *.yaml alert rule files: the Loki rules were named .yaml while only *.yml was loaded, so they silently never ran. Fix the Grafana domain typo (writefo.fun -> writefor.fun). Comment out the postgres-playground datasource, which used the Grafana admin password for a role that has no password and could never authenticate. --- .../idols-aquamarine/grafana/datasources.nix | 56 ++++++++++--------- hosts/idols-aquamarine/grafana/default.nix | 2 +- hosts/idols-aquamarine/monitoring/alert.nix | 2 + 3 files changed, 34 insertions(+), 26 deletions(-) diff --git a/hosts/idols-aquamarine/grafana/datasources.nix b/hosts/idols-aquamarine/grafana/datasources.nix index 2f70fc19..0451baf7 100644 --- a/hosts/idols-aquamarine/grafana/datasources.nix +++ b/hosts/idols-aquamarine/grafana/datasources.nix @@ -88,31 +88,37 @@ }; editable = false; } - { - # https://grafana.com/docs/grafana/latest/datasources/postgres/configure/ - name = "postgres-playground"; - type = "postgres"; - url = "postgres.writefor.fun:5432"; - user = "playground"; - secureJsonData = { - password = "$__file{${config.age.secrets."grafana-admin-password".path}}"; - }; - jsonData = { - database = "playground"; - sslmode = "verify-full"; # disable/require/verify-ca/verify-full - maxOpenConns = 50; - maxIdleConns = 250; - maxIdleConnsAuto = true; - connMaxLifetime = 14400; - timeInterval = "1m"; - timescaledb = false; - postgresVersion = 1500; # 15.xx - # tls - tlsConfigurationMethod = "file-path"; - sslRootCertFile = ../../../certs/ecc-ca.crt; - }; - editable = false; - } + # Disabled: the `playground` role has no password and remote logins + # require scram-sha-256, so this datasource could never authenticate. + # Re-enable once a `playground` DB password is provisioned + # (docs/system-audit-2026-09-19.md, item 1.5). + /* + { + # https://grafana.com/docs/grafana/latest/datasources/postgres/configure/ + name = "postgres-playground"; + type = "postgres"; + url = "postgres.writefor.fun:5432"; + user = "playground"; + secureJsonData = { + password = "$__file{${config.age.secrets."postgres-playground-password".path}}"; + }; + jsonData = { + database = "playground"; + sslmode = "verify-full"; # disable/require/verify-ca/verify-full + maxOpenConns = 50; + maxIdleConns = 250; + maxIdleConnsAuto = true; + connMaxLifetime = 14400; + timeInterval = "1m"; + timescaledb = false; + postgresVersion = 1500; # 15.xx + # tls + tlsConfigurationMethod = "file-path"; + sslRootCertFile = ../../../certs/ecc-ca.crt; + }; + editable = false; + } + */ { name = "infinity-dataviewer"; type = "yesoreyeram-infinity-datasource"; diff --git a/hosts/idols-aquamarine/grafana/default.nix b/hosts/idols-aquamarine/grafana/default.nix index df5c0ec4..4492eeb9 100644 --- a/hosts/idols-aquamarine/grafana/default.nix +++ b/hosts/idols-aquamarine/grafana/default.nix @@ -20,7 +20,7 @@ http_addr = "127.0.0.1"; http_port = 3351; protocol = "http"; - domain = "grafana.writefo.fun"; + domain = "grafana.writefor.fun"; # Redirect to correct domain if the host header does not match the domain. Prevents DNS rebinding attacks. serve_from_sub_path = false; # Add subpath to the root_url if serve_from_sub_path is true diff --git a/hosts/idols-aquamarine/monitoring/alert.nix b/hosts/idols-aquamarine/monitoring/alert.nix index 00c6de36..ef5deee5 100644 --- a/hosts/idols-aquamarine/monitoring/alert.nix +++ b/hosts/idols-aquamarine/monitoring/alert.nix @@ -20,7 +20,9 @@ # Path to the files with alerting and/or recording rules. rule = [ "${./alert_rules}/*.yml" + "${./alert_rules}/*.yaml" "${./recoding_rules}/*.yml" + "${./recoding_rules}/*.yaml" ]; # https://docs.victoriametrics.com/victoriametrics/vmalert/#link-to-alert-source # Set this two args to generate the correct `.GeneratorURL` From fc8b9e5ba01342280498ac36232fce3bf569490d Mon Sep 17 00:00:00 2001 From: Ryan Yin Date: Sat, 19 Sep 2026 12:11:11 +0800 Subject: [PATCH 3/6] chore(aquamarine): disable the broken restic backup The restic job needs three agenix secrets that do not exist yet (restic-password, ssh-key-for-restic-backup and an rclone config for the smb-downloads remote). Comment it out until they are provisioned so it does not fail silently every night. --- hosts/idols-aquamarine/restic.nix | 132 ++++++++++++++++-------------- 1 file changed, 69 insertions(+), 63 deletions(-) diff --git a/hosts/idols-aquamarine/restic.nix b/hosts/idols-aquamarine/restic.nix index b3e70033..5f881b24 100644 --- a/hosts/idols-aquamarine/restic.nix +++ b/hosts/idols-aquamarine/restic.nix @@ -5,77 +5,83 @@ let rcloneConfigFile = "/etc/agenix/rclone-conf-for-restic-backup"; in { + # DISABLED: this backup has never worked. It needs three agenix secrets that + # do not exist in nix-secrets yet (`restic-password`, `ssh-key-for-restic-backup`, + # and an rclone config with the `smb-downloads` remote; the existing + # `rclone.conf` secret is only defined for the desktop secret set). Re-enable + # once the secrets are provisioned. + # # https://github.com/NixOS/nixpkgs/blob/nixos-26.05/nixos/modules/services/backup/restic.nix - services.restic.backups = { - homelab-backup = { - inherit passwordFile; - initialize = true; # Initialize the repository if it doesn't exist. - repository = "rclone:smb-downloads:/Downloads/kubevirt-backup/"; # backup to a rclone remote + /* + services.restic.backups = { + homelab-backup = { + inherit passwordFile; + initialize = true; # Initialize the repository if it doesn't exist. + repository = "rclone:smb-downloads:/Downloads/kubevirt-backup/"; # backup to a rclone remote - # rclone related - # rcloneOptions = { - # bwlimit = "100M"; # Limit the bandwidth used by rclone. - # }; - inherit rcloneConfigFile; + # rclone related + # rcloneOptions = { + # bwlimit = "100M"; # Limit the bandwidth used by rclone. + # }; + inherit rcloneConfigFile; - # Which local paths to backup, in addition to ones specified via `dynamicFilesFrom`. - paths = [ - "/tmp/restic-backup-temp" - ]; - # - # A script that produces a list of files to back up. The - # results of this command are given to the '--files-from' - # option. The result is merged with paths specified via `paths`. - # dynamicFilesFrom = "find /home/matt/git -type d -name .git"; - # - # Patterns to exclude when backing up. See - # https://restic.readthedocs.io/en/latest/040_backup.html#excluding-files - # for details on syntax. - exclude = [ ]; + # Which local paths to backup, in addition to ones specified via `dynamicFilesFrom`. + paths = [ + "/tmp/restic-backup-temp" + ]; + # + # A script that produces a list of files to back up. The + # results of this command are given to the '--files-from' + # option. The result is merged with paths specified via `paths`. + # dynamicFilesFrom = "find /home/matt/git -type d -name .git"; + # + # Patterns to exclude when backing up. See + # https://restic.readthedocs.io/en/latest/040_backup.html#excluding-files + # for details on syntax. + exclude = [ ]; - # A script that must run before starting the backup process. - backupPrepareCommand = '' - ${pkgs.nushell}/bin/nu -c ' - let kubevirt_nodes = [ - "kubevirt-shoryu" - "kubevirt-shushou" - "kubevirt-youko" - ] + # A script that must run before starting the backup process. + backupPrepareCommand = '' + ${pkgs.nushell}/bin/nu -c ' + let kubevirt_nodes = [ + "kubevirt-shoryu" + "kubevirt-shushou" + "kubevirt-youko" + ] - kubevirt_nodes | each {|it| - rsync -avz \ - -e "ssh -i ${sshKeyPath}" \ - $"($it):/perissitent/" $"/tmp/restic-backup-temp/($it)" - } - ' - ''; - # A script that must run after finishing the backup process. - backupCleanupCommand = "rm -rf /tmp/restic-backup-temp"; + kubevirt_nodes | each {|it| + ^${pkgs.rsync}/bin/rsync -avz -e $"ssh -i ${sshKeyPath}" $"($it):/persistent/" $"/tmp/restic-backup-temp/($it)" + } + ' + ''; + # A script that must run after finishing the backup process. + backupCleanupCommand = "rm -rf /tmp/restic-backup-temp"; - # Extra extended options to be passed to the restic --option flag. - # extraOptions = []; + # Extra extended options to be passed to the restic --option flag. + # extraOptions = []; - # Extra arguments passed to restic backup. - # extraBackupArgs = [ - # "--exclude-file=/etc/restic/excludes-list" - # ]; + # Extra arguments passed to restic backup. + # extraBackupArgs = [ + # "--exclude-file=/etc/restic/excludes-list" + # ]; - # repository = "/mnt/backup-hdd"; # backup to a local directory - # When to run the backup. See {manpage}`systemd.timer(5)` for details. - timerConfig = { - OnCalendar = "01:30"; - RandomizedDelaySec = "1h"; + # repository = "/mnt/backup-hdd"; # backup to a local directory + # When to run the backup. See {manpage}`systemd.timer(5)` for details. + timerConfig = { + OnCalendar = "01:30"; + RandomizedDelaySec = "1h"; + }; + # A list of options (--keep-* et al.) for 'restic forget --prune', + # to automatically prune old snapshots. + # The 'forget' command is run *after* the 'backup' command, so + # keep that in mind when constructing the --keep-* options. + pruneOpts = [ + "--keep-daily 3" + "--keep-weekly 3" + "--keep-monthly 3" + "--keep-yearly 3" + ]; }; - # A list of options (--keep-* et al.) for 'restic forget --prune', - # to automatically prune old snapshots. - # The 'forget' command is run *after* the 'backup' command, so - # keep that in mind when constructing the --keep-* options. - pruneOpts = [ - "--keep-daily 3" - "--keep-weekly 3" - "--keep-monthly 3" - "--keep-yearly 3" - ]; }; - }; + */ } From 3318103f8a8cdcb4e2bc9a9915757ac8dbe9284e Mon Sep 17 00:00:00 2001 From: Ryan Yin Date: Sat, 19 Sep 2026 12:11:15 +0800 Subject: [PATCH 4/6] chore(guix): remove the unused Guix integration Guix was enabled on every desktop host, but shoukei had no /gnu mount (store and state lived on the tmpfs root) and it was not used. Drop the module, the modules.guix option, the @guix subvolume and the /var/guix preservation entry. --- hosts/idols-ai/disko-fs.nix | 7 ------- hosts/idols-ai/hardware-configuration.nix | 2 +- modules/nixos/desktop/guix.nix | 21 --------------------- 3 files changed, 1 insertion(+), 29 deletions(-) delete mode 100644 modules/nixos/desktop/guix.nix diff --git a/hosts/idols-ai/disko-fs.nix b/hosts/idols-ai/disko-fs.nix index 0497631f..04e5bdf8 100644 --- a/hosts/idols-ai/disko-fs.nix +++ b/hosts/idols-ai/disko-fs.nix @@ -87,13 +87,6 @@ "noatime" ]; }; - "@guix" = { - mountpoint = "/gnu"; - mountOptions = [ - "compress-force=zstd:1" - "noatime" - ]; - }; "@persistent" = { mountpoint = "/persistent"; mountOptions = [ diff --git a/hosts/idols-ai/hardware-configuration.nix b/hosts/idols-ai/hardware-configuration.nix index d9ed6f0a..9b7cf978 100644 --- a/hosts/idols-ai/hardware-configuration.nix +++ b/hosts/idols-ai/hardware-configuration.nix @@ -74,7 +74,7 @@ "exfat" ]; - # LUKS initrd, all fileSystems (/, /boot, /btr_pool, /nix, /gnu, /persistent, /snapshots, /tmp, /swap) + # LUKS initrd, all fileSystems (/, /boot, /btr_pool, /nix, /persistent, /snapshots, /tmp, /swap) # and swap (including /swap/swapfile bind and swapDevices) are managed by disko (disko-fs.nix). # Enables DHCP on each ethernet and wireless interface. In case of scripted networking diff --git a/modules/nixos/desktop/guix.nix b/modules/nixos/desktop/guix.nix deleted file mode 100644 index d8136ccd..00000000 --- a/modules/nixos/desktop/guix.nix +++ /dev/null @@ -1,21 +0,0 @@ -{ - # https://github.com/NixOS/nixpkgs/blob/master/nixos/modules/services/misc/guix/default.nix - services.guix = { - enable = true; - # The store directory where the Guix service will serve to/from. - # NOTE: most of the cached builds are assumed to be in `/gnu/store`. - storeDir = "/gnu/store"; - # The state directory where Guix service will store its data such as its - # user-specific profiles, cache, and state files. - stateDir = "/var"; - gc = { - enable = true; - # https://guix.gnu.org/en/manual/en/html_node/Invoking-guix-gc.html - extraArgs = [ - "--delete-generations=1m" - "--free-space=10G" - "--optimize" - ]; - }; - }; -} From a8ea040c324db70bb99e2729eacfda3a9c942b37 Mon Sep 17 00:00:00 2001 From: Ryan Yin Date: Sat, 19 Sep 2026 12:11:18 +0800 Subject: [PATCH 5/6] fix(trash): persist the cleanup timer trash-empty lacked Persistent=true, so a host that was off or suspended at the scheduled time never ran the purge and trash retention silently stopped. --- modules/nixos/base/trash.nix | 2 ++ 1 file changed, 2 insertions(+) diff --git a/modules/nixos/base/trash.nix b/modules/nixos/base/trash.nix index 1c70598c..d67ae950 100644 --- a/modules/nixos/base/trash.nix +++ b/modules/nixos/base/trash.nix @@ -56,6 +56,8 @@ in timerConfig = { OnCalendar = "daily"; RandomizedDelaySec = "15min"; + # run the skipped purge on the next boot if the host was off/suspended. + Persistent = true; }; }; } From f23d5d4245ca6e74982498e39106ab0e937d4f4f Mon Sep 17 00:00:00 2001 From: Ryan Yin Date: Sat, 19 Sep 2026 12:14:20 +0800 Subject: [PATCH 6/6] docs(monitoring): note dead-man switch and backup freshness TODOs --- hosts/idols-aquamarine/monitoring/README.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/hosts/idols-aquamarine/monitoring/README.md b/hosts/idols-aquamarine/monitoring/README.md index d4ed16d2..3d36a4c5 100644 --- a/hosts/idols-aquamarine/monitoring/README.md +++ b/hosts/idols-aquamarine/monitoring/README.md @@ -6,3 +6,16 @@ - Collection of Prometheus alerting rules. - [victoria-metrics-k8s-stack/files/rules](https://github.com/VictoriaMetrics/helm-charts/tree/master/charts/victoria-metrics-k8s-stack/files/rules/generated) - Alert Rules & Recoding Rules used by kube-prometheus-stack. + +## TODO + +- **Dead-man switch**: `Watchdog` in `alert_rules/general.yml` always fires (`vector(1)`) but is + routed to the `null` receiver in `alert.nix`, so a dead alerting pipeline looks identical to a + healthy one. Route it to an external heartbeat service (healthchecks.io, Uptime Kuma, PagerDuty + DeadMansSnitch) that notifies when the heartbeat stops, or run an out-of-band probe against + Alertmanager. +- **Backup freshness**: nothing alerts on "no recent successful backup". The systemd `failed` state + only catches crashes, not a job that never ran or a backup that is silently stale. Export the last + successful backup timestamp (restic prometheus exporter; btrfs snapshot time via the node-exporter + textfile collector) and alert when it exceeds the SLO, and add an off-host btrbk `target` so local + snapshots are not the only copy.