diff --git a/hosts/idols-ai/disko-fs.nix b/hosts/idols-ai/disko-fs.nix index 0497631f..04e5bdf8 100644 --- a/hosts/idols-ai/disko-fs.nix +++ b/hosts/idols-ai/disko-fs.nix @@ -87,13 +87,6 @@ "noatime" ]; }; - "@guix" = { - mountpoint = "/gnu"; - mountOptions = [ - "compress-force=zstd:1" - "noatime" - ]; - }; "@persistent" = { mountpoint = "/persistent"; mountOptions = [ diff --git a/hosts/idols-ai/hardware-configuration.nix b/hosts/idols-ai/hardware-configuration.nix index d9ed6f0a..9b7cf978 100644 --- a/hosts/idols-ai/hardware-configuration.nix +++ b/hosts/idols-ai/hardware-configuration.nix @@ -74,7 +74,7 @@ "exfat" ]; - # LUKS initrd, all fileSystems (/, /boot, /btr_pool, /nix, /gnu, /persistent, /snapshots, /tmp, /swap) + # LUKS initrd, all fileSystems (/, /boot, /btr_pool, /nix, /persistent, /snapshots, /tmp, /swap) # and swap (including /swap/swapfile bind and swapDevices) are managed by disko (disko-fs.nix). # Enables DHCP on each ethernet and wireless interface. In case of scripted networking diff --git a/hosts/idols-aquamarine/grafana/datasources.nix b/hosts/idols-aquamarine/grafana/datasources.nix index 2f70fc19..0451baf7 100644 --- a/hosts/idols-aquamarine/grafana/datasources.nix +++ b/hosts/idols-aquamarine/grafana/datasources.nix @@ -88,31 +88,37 @@ }; editable = false; } - { - # https://grafana.com/docs/grafana/latest/datasources/postgres/configure/ - name = "postgres-playground"; - type = "postgres"; - url = "postgres.writefor.fun:5432"; - user = "playground"; - secureJsonData = { - password = "$__file{${config.age.secrets."grafana-admin-password".path}}"; - }; - jsonData = { - database = "playground"; - sslmode = "verify-full"; # disable/require/verify-ca/verify-full - maxOpenConns = 50; - maxIdleConns = 250; - maxIdleConnsAuto = true; - connMaxLifetime = 14400; - timeInterval = "1m"; - timescaledb = false; - postgresVersion = 1500; # 15.xx - # tls - tlsConfigurationMethod = "file-path"; - sslRootCertFile = ../../../certs/ecc-ca.crt; - }; - editable = false; - } + # Disabled: the `playground` role has no password and remote logins + # require scram-sha-256, so this datasource could never authenticate. + # Re-enable once a `playground` DB password is provisioned + # (docs/system-audit-2026-09-19.md, item 1.5). + /* + { + # https://grafana.com/docs/grafana/latest/datasources/postgres/configure/ + name = "postgres-playground"; + type = "postgres"; + url = "postgres.writefor.fun:5432"; + user = "playground"; + secureJsonData = { + password = "$__file{${config.age.secrets."postgres-playground-password".path}}"; + }; + jsonData = { + database = "playground"; + sslmode = "verify-full"; # disable/require/verify-ca/verify-full + maxOpenConns = 50; + maxIdleConns = 250; + maxIdleConnsAuto = true; + connMaxLifetime = 14400; + timeInterval = "1m"; + timescaledb = false; + postgresVersion = 1500; # 15.xx + # tls + tlsConfigurationMethod = "file-path"; + sslRootCertFile = ../../../certs/ecc-ca.crt; + }; + editable = false; + } + */ { name = "infinity-dataviewer"; type = "yesoreyeram-infinity-datasource"; diff --git a/hosts/idols-aquamarine/grafana/default.nix b/hosts/idols-aquamarine/grafana/default.nix index df5c0ec4..4492eeb9 100644 --- a/hosts/idols-aquamarine/grafana/default.nix +++ b/hosts/idols-aquamarine/grafana/default.nix @@ -20,7 +20,7 @@ http_addr = "127.0.0.1"; http_port = 3351; protocol = "http"; - domain = "grafana.writefo.fun"; + domain = "grafana.writefor.fun"; # Redirect to correct domain if the host header does not match the domain. Prevents DNS rebinding attacks. serve_from_sub_path = false; # Add subpath to the root_url if serve_from_sub_path is true diff --git a/hosts/idols-aquamarine/monitoring/README.md b/hosts/idols-aquamarine/monitoring/README.md index d4ed16d2..3d36a4c5 100644 --- a/hosts/idols-aquamarine/monitoring/README.md +++ b/hosts/idols-aquamarine/monitoring/README.md @@ -6,3 +6,16 @@ - Collection of Prometheus alerting rules. - [victoria-metrics-k8s-stack/files/rules](https://github.com/VictoriaMetrics/helm-charts/tree/master/charts/victoria-metrics-k8s-stack/files/rules/generated) - Alert Rules & Recoding Rules used by kube-prometheus-stack. + +## TODO + +- **Dead-man switch**: `Watchdog` in `alert_rules/general.yml` always fires (`vector(1)`) but is + routed to the `null` receiver in `alert.nix`, so a dead alerting pipeline looks identical to a + healthy one. Route it to an external heartbeat service (healthchecks.io, Uptime Kuma, PagerDuty + DeadMansSnitch) that notifies when the heartbeat stops, or run an out-of-band probe against + Alertmanager. +- **Backup freshness**: nothing alerts on "no recent successful backup". The systemd `failed` state + only catches crashes, not a job that never ran or a backup that is silently stale. Export the last + successful backup timestamp (restic prometheus exporter; btrfs snapshot time via the node-exporter + textfile collector) and alert when it exceeds the SLO, and add an off-host btrbk `target` so local + snapshots are not the only copy. diff --git a/hosts/idols-aquamarine/monitoring/alert.nix b/hosts/idols-aquamarine/monitoring/alert.nix index 00c6de36..ef5deee5 100644 --- a/hosts/idols-aquamarine/monitoring/alert.nix +++ b/hosts/idols-aquamarine/monitoring/alert.nix @@ -20,7 +20,9 @@ # Path to the files with alerting and/or recording rules. rule = [ "${./alert_rules}/*.yml" + "${./alert_rules}/*.yaml" "${./recoding_rules}/*.yml" + "${./recoding_rules}/*.yaml" ]; # https://docs.victoriametrics.com/victoriametrics/vmalert/#link-to-alert-source # Set this two args to generate the correct `.GeneratorURL` diff --git a/hosts/idols-aquamarine/restic.nix b/hosts/idols-aquamarine/restic.nix index b3e70033..5f881b24 100644 --- a/hosts/idols-aquamarine/restic.nix +++ b/hosts/idols-aquamarine/restic.nix @@ -5,77 +5,83 @@ let rcloneConfigFile = "/etc/agenix/rclone-conf-for-restic-backup"; in { + # DISABLED: this backup has never worked. It needs three agenix secrets that + # do not exist in nix-secrets yet (`restic-password`, `ssh-key-for-restic-backup`, + # and an rclone config with the `smb-downloads` remote; the existing + # `rclone.conf` secret is only defined for the desktop secret set). Re-enable + # once the secrets are provisioned. + # # https://github.com/NixOS/nixpkgs/blob/nixos-26.05/nixos/modules/services/backup/restic.nix - services.restic.backups = { - homelab-backup = { - inherit passwordFile; - initialize = true; # Initialize the repository if it doesn't exist. - repository = "rclone:smb-downloads:/Downloads/kubevirt-backup/"; # backup to a rclone remote + /* + services.restic.backups = { + homelab-backup = { + inherit passwordFile; + initialize = true; # Initialize the repository if it doesn't exist. + repository = "rclone:smb-downloads:/Downloads/kubevirt-backup/"; # backup to a rclone remote - # rclone related - # rcloneOptions = { - # bwlimit = "100M"; # Limit the bandwidth used by rclone. - # }; - inherit rcloneConfigFile; + # rclone related + # rcloneOptions = { + # bwlimit = "100M"; # Limit the bandwidth used by rclone. + # }; + inherit rcloneConfigFile; - # Which local paths to backup, in addition to ones specified via `dynamicFilesFrom`. - paths = [ - "/tmp/restic-backup-temp" - ]; - # - # A script that produces a list of files to back up. The - # results of this command are given to the '--files-from' - # option. The result is merged with paths specified via `paths`. - # dynamicFilesFrom = "find /home/matt/git -type d -name .git"; - # - # Patterns to exclude when backing up. See - # https://restic.readthedocs.io/en/latest/040_backup.html#excluding-files - # for details on syntax. - exclude = [ ]; + # Which local paths to backup, in addition to ones specified via `dynamicFilesFrom`. + paths = [ + "/tmp/restic-backup-temp" + ]; + # + # A script that produces a list of files to back up. The + # results of this command are given to the '--files-from' + # option. The result is merged with paths specified via `paths`. + # dynamicFilesFrom = "find /home/matt/git -type d -name .git"; + # + # Patterns to exclude when backing up. See + # https://restic.readthedocs.io/en/latest/040_backup.html#excluding-files + # for details on syntax. + exclude = [ ]; - # A script that must run before starting the backup process. - backupPrepareCommand = '' - ${pkgs.nushell}/bin/nu -c ' - let kubevirt_nodes = [ - "kubevirt-shoryu" - "kubevirt-shushou" - "kubevirt-youko" - ] + # A script that must run before starting the backup process. + backupPrepareCommand = '' + ${pkgs.nushell}/bin/nu -c ' + let kubevirt_nodes = [ + "kubevirt-shoryu" + "kubevirt-shushou" + "kubevirt-youko" + ] - kubevirt_nodes | each {|it| - rsync -avz \ - -e "ssh -i ${sshKeyPath}" \ - $"($it):/perissitent/" $"/tmp/restic-backup-temp/($it)" - } - ' - ''; - # A script that must run after finishing the backup process. - backupCleanupCommand = "rm -rf /tmp/restic-backup-temp"; + kubevirt_nodes | each {|it| + ^${pkgs.rsync}/bin/rsync -avz -e $"ssh -i ${sshKeyPath}" $"($it):/persistent/" $"/tmp/restic-backup-temp/($it)" + } + ' + ''; + # A script that must run after finishing the backup process. + backupCleanupCommand = "rm -rf /tmp/restic-backup-temp"; - # Extra extended options to be passed to the restic --option flag. - # extraOptions = []; + # Extra extended options to be passed to the restic --option flag. + # extraOptions = []; - # Extra arguments passed to restic backup. - # extraBackupArgs = [ - # "--exclude-file=/etc/restic/excludes-list" - # ]; + # Extra arguments passed to restic backup. + # extraBackupArgs = [ + # "--exclude-file=/etc/restic/excludes-list" + # ]; - # repository = "/mnt/backup-hdd"; # backup to a local directory - # When to run the backup. See {manpage}`systemd.timer(5)` for details. - timerConfig = { - OnCalendar = "01:30"; - RandomizedDelaySec = "1h"; + # repository = "/mnt/backup-hdd"; # backup to a local directory + # When to run the backup. See {manpage}`systemd.timer(5)` for details. + timerConfig = { + OnCalendar = "01:30"; + RandomizedDelaySec = "1h"; + }; + # A list of options (--keep-* et al.) for 'restic forget --prune', + # to automatically prune old snapshots. + # The 'forget' command is run *after* the 'backup' command, so + # keep that in mind when constructing the --keep-* options. + pruneOpts = [ + "--keep-daily 3" + "--keep-weekly 3" + "--keep-monthly 3" + "--keep-yearly 3" + ]; }; - # A list of options (--keep-* et al.) for 'restic forget --prune', - # to automatically prune old snapshots. - # The 'forget' command is run *after* the 'backup' command, so - # keep that in mind when constructing the --keep-* options. - pruneOpts = [ - "--keep-daily 3" - "--keep-weekly 3" - "--keep-monthly 3" - "--keep-yearly 3" - ]; }; - }; + */ } diff --git a/modules/nixos/base/trash.nix b/modules/nixos/base/trash.nix index 1c70598c..d67ae950 100644 --- a/modules/nixos/base/trash.nix +++ b/modules/nixos/base/trash.nix @@ -56,6 +56,8 @@ in timerConfig = { OnCalendar = "daily"; RandomizedDelaySec = "15min"; + # run the skipped purge on the next boot if the host was off/suspended. + Persistent = true; }; }; } diff --git a/modules/nixos/desktop/guix.nix b/modules/nixos/desktop/guix.nix deleted file mode 100644 index d8136ccd..00000000 --- a/modules/nixos/desktop/guix.nix +++ /dev/null @@ -1,21 +0,0 @@ -{ - # https://github.com/NixOS/nixpkgs/blob/master/nixos/modules/services/misc/guix/default.nix - services.guix = { - enable = true; - # The store directory where the Guix service will serve to/from. - # NOTE: most of the cached builds are assumed to be in `/gnu/store`. - storeDir = "/gnu/store"; - # The state directory where Guix service will store its data such as its - # user-specific profiles, cache, and state files. - stateDir = "/var"; - gc = { - enable = true; - # https://guix.gnu.org/en/manual/en/html_node/Invoking-guix-gc.html - extraArgs = [ - "--delete-generations=1m" - "--free-space=10G" - "--optimize" - ]; - }; - }; -} diff --git a/outputs/default.nix b/outputs/default.nix index 6f8404c1..4b66723b 100644 --- a/outputs/default.nix +++ b/outputs/default.nix @@ -154,8 +154,19 @@ in evalTests = lib.lists.all (it: it.evalTests == { }) allSystemValues; checks = forAllSystems (system: { - # eval-tests per system - eval-tests = allSystems.${system}.evalTests == { }; + # eval-tests per system. `nix flake check` requires every check to be a + # derivation, so wrap the boolean result in one instead of returning a bool. + eval-tests = + let + pkgs = nixpkgs.legacyPackages.${system}; + results = allSystems.${system}.evalTests; + in + pkgs.runCommand "eval-tests" { } ( + if results == { } then + "touch $out" + else + "echo 'eval tests failed: evalTests is not empty' >&2; exit 1" + ); pre-commit-check = pre-commit-hooks.lib.${system}.run { src = mylib.relativeToRoot ".";