mirror of
https://github.com/ryan4yin/nix-config.git
synced 2026-11-18 05:58:17 +01:00
feat(nixos): Nix store hygiene, PostgreSQL dumps, OOM and backup quotas (#532)
Assisted-by: DeepSeek Harness (claude-opus-5-5)
This commit is contained in:
1 parent
9b1a716aa8
commit
667fd76cac
6 files changed
+88
-16
No files matched your search
+19
-8
@@ -25,7 +25,11 @@ How this fleet is backed up, where the data lives, and how to restore it.
|
||||
automatic retention it needs. A compromised desktop can still `forget --prune` its own repository
|
||||
through the REST API, so youko also snapshots `/data/backups` hourly into a read-only btrfs
|
||||
subvolume the desktop cannot reach (see
|
||||
`hosts/12kingdoms-youko/homelab-services/restic-server-snapshots.nix`).
|
||||
`hosts/12kingdoms-youko/homelab-services/restic-server-snapshots.nix`). The REST store has one
|
||||
512 GiB quota (`--max-size`) shared by all repositories, so a runaway client cannot fill the HDD
|
||||
pool it shares with `/data/apps` and `/data/fileshare`; at the quota, uploads fail with HTTP 507
|
||||
and the client's backup-failure alert fires. youko's own repository (`/data/backups/restic`) is
|
||||
written directly, outside the quota.
|
||||
3. **cloud copy** (planned). youko will copy the homelab repositories with `restic copy` (it holds
|
||||
that password). Desktops copy their own: youko does not hold the desktop password. This layer is
|
||||
what protects history against a compromised host, so the target should be immutable or versioned
|
||||
@@ -60,13 +64,13 @@ Consequences worth remembering:
|
||||
Slots are assigned per host; the random delay only smooths load and must not be relied on to prevent
|
||||
collisions.
|
||||
|
||||
| Time | Job |
|
||||
| ---------------- | ----------------------------------- |
|
||||
| 01:30 + 15m | desktops push to youko |
|
||||
| 02:30 + 15m | youko backs itself up |
|
||||
| hourly | youko snapshots the REST store |
|
||||
| 03:45:20 Tue/Sat | btrbk local snapshots |
|
||||
| 06:00 (planned) | youko copies homelab repos to cloud |
|
||||
| Time | Job |
|
||||
| ---------------- | -------------------------------------------- |
|
||||
| 01:30 + 15m | desktops push to youko |
|
||||
| 02:30 + 15m | youko dumps PostgreSQL, then backs itself up |
|
||||
| hourly | youko snapshots the REST store |
|
||||
| 03:45:20 Tue/Sat | btrbk local snapshots |
|
||||
| 06:00 (planned) | youko copies homelab repos to cloud |
|
||||
|
||||
Set per host via `modules.restic-backup.onCalendar` and `randomizedDelaySec`.
|
||||
|
||||
@@ -129,6 +133,12 @@ Restoring a btrbk snapshot (offline; stop writers first):
|
||||
- Units: `systemctl status restic-backups-homelab`, `systemctl list-timers 'restic-backups*'`.
|
||||
- Repository integrity (where the password is): `restic check`, which runs after every backup via
|
||||
`modules.restic-backup.checkOpts` (a 5% data read, `1/20`, by default).
|
||||
- PostgreSQL on youko: `postgresqlBackup.service` runs before every `restic-backups-homelab` run and
|
||||
writes `pg_dumpall` output to `/var/lib/postgresql-backup/all.sql.zstd` (the previous dump is kept
|
||||
as `all.prev.sql.zstd`). Restore into an empty cluster with
|
||||
`zstd -dc all.sql.zstd | sudo -u postgres psql -X postgres`. A failed dump does not block the
|
||||
backup; it shows up as a failed unit.
|
||||
- REST store quota: `restic-rest-server` logs `Quota initialized, currently using N GiB` at start.
|
||||
- Server side, ciphertext only: repository directories under `/data/backups/rest-server/<user>/`,
|
||||
with no leftover files in `locks/` and no stale `.tmp` files.
|
||||
- `restic-rest-server`'s log must not print `Invalid htpasswd entry`: that means the client's REST
|
||||
@@ -158,6 +168,7 @@ Restoring a btrbk snapshot (offline; stop writers first):
|
||||
| [`modules/nixos/base/btrbk.nix`](../modules/nixos/base/btrbk.nix) | local btrfs snapshots |
|
||||
| `hosts/12kingdoms-youko/default.nix` | the REST server and youko's own backup |
|
||||
| `hosts/12kingdoms-youko/homelab-services/restic-server-snapshots.nix` | hourly read-only snapshots of the REST store |
|
||||
| `hosts/12kingdoms-youko/homelab-services/postgresql.nix` | the PostgreSQL dump taken before youko's backup |
|
||||
| `hosts/12kingdoms-youko/homelab-services/caddy.nix` | the `restic.writefor.fun` vhost |
|
||||
| `secrets/nixos.nix` | which secret is defined on which host |
|
||||
| `hosts/idols-ai/restic.nix`, `hosts/12kingdoms-youko/default.nix` | per-host backup configuration |
|
||||
@@ -94,7 +94,13 @@ in
|
||||
# expose /metrics for victoriametrics; unauthenticated is fine because the
|
||||
# server only listens on loopback
|
||||
prometheus = true;
|
||||
extraFlags = [ "--prometheus-no-auth" ];
|
||||
extraFlags = [
|
||||
"--prometheus-no-auth"
|
||||
# One quota for the whole store, all repositories together, so a runaway
|
||||
# client cannot fill the HDD pool shared with /data/apps and /data/fileshare.
|
||||
"--max-size"
|
||||
(toString (512 * 1024 * 1024 * 1024)) # 512 GiB
|
||||
];
|
||||
};
|
||||
|
||||
# repositories live on the HDD, which has to be mounted first
|
||||
|
||||
@@ -63,14 +63,15 @@ document is the agreed design and the rationale.
|
||||
|
||||
## Backup and disaster recovery
|
||||
|
||||
- **Database**: already covered by the existing youko restic backup, which snapshots
|
||||
`/btr_pool/@persistent` (Postgres data lives there).
|
||||
- **Database**: covered by the youko restic backup, which snapshots `/btr_pool/@persistent`. Each
|
||||
run first writes a `pg_dumpall` dump to `/var/lib/postgresql-backup` (see
|
||||
[BACKUP.md](../../../../docs/BACKUP.md#verifying)); restore from that dump, since the raw data
|
||||
directory in the snapshot is only crash-consistent.
|
||||
- **Media and existing photos**: both live on the HDD already. A local HDD->HDD backup adds no
|
||||
protection, so none is added here. This is deferred to the planned cloud copy, which will cover
|
||||
the photo library.
|
||||
- **Note for the cloud-copy work**: a raw btrfs snapshot of a running Postgres is only
|
||||
crash-consistent; prefer a `pg_dump` into the cloud set for a clean restore. That set should
|
||||
include `mediaLocation` and the external-library source.
|
||||
- **Note for the cloud-copy work**: that set should include `mediaLocation` and the external-library
|
||||
source.
|
||||
- **One-time disruption**: adding the extensions and `shared_preload_libraries` restarts PostgreSQL
|
||||
(affects the `playground` DB only).
|
||||
|
||||
|
||||
@@ -57,7 +57,12 @@ in
|
||||
|
||||
# logging
|
||||
log_connections = true;
|
||||
log_statement = "all";
|
||||
# Log slow statements instead of every statement; also lock waits longer
|
||||
# than deadlock_timeout, every temp file, and slow autovacuum runs.
|
||||
log_min_duration_statement = 1000; # ms
|
||||
log_lock_waits = true;
|
||||
log_temp_files = 0;
|
||||
log_autovacuum_min_duration = 1000; # ms
|
||||
logging_collector = true;
|
||||
log_disconnections = true;
|
||||
log_destination = lib.mkForce "syslog";
|
||||
@@ -119,6 +124,21 @@ in
|
||||
# '';
|
||||
};
|
||||
|
||||
# Logical dump of all databases and roles. The restic snapshot of the data
|
||||
# directory is only crash-consistent; the dump restores cleanly. /var/lib is
|
||||
# persisted, so the dump lands in the @persistent tree that restic backs up.
|
||||
services.postgresqlBackup = {
|
||||
enable = true;
|
||||
location = "/var/lib/postgresql-backup";
|
||||
compression = "zstd";
|
||||
# no timer of its own: each restic run takes a fresh dump first
|
||||
startAt = [ ];
|
||||
};
|
||||
systemd.services.restic-backups-homelab = {
|
||||
wants = [ "postgresqlBackup.service" ];
|
||||
after = [ "postgresqlBackup.service" ];
|
||||
};
|
||||
|
||||
services.prometheus.exporters.postgres = {
|
||||
enable = true;
|
||||
# loopback only: scraped by VictoriaMetrics on the same host
|
||||
|
||||
+12
-1
@@ -62,7 +62,18 @@ in
|
||||
|
||||
# zram itself is provided by modules/nixos/base/zram.nix (enabled by default).
|
||||
# Kill the greediest process before the host starts thrashing / gets OOM-killed.
|
||||
services.earlyoom.enable = true;
|
||||
services.earlyoom = {
|
||||
enable = true;
|
||||
# The MicroVMs' qemu and virtiofsd are the largest processes and would be
|
||||
# killed first. Prefixes of /proc/PID/comm, which is cut to 15 characters
|
||||
# (wrapped binaries start with a dot).
|
||||
extraArgs = [
|
||||
"--avoid"
|
||||
"^(\\.qemu-system-x8|virtiofsd|\\.postgres-wrap|victoria-metric)"
|
||||
];
|
||||
};
|
||||
# Builds inherit the daemon's oom_score_adj, so earlyoom and the kernel kill them first.
|
||||
systemd.services.nix-daemon.serviceConfig.OOMScoreAdjust = 500;
|
||||
|
||||
environment.systemPackages = with pkgs; [
|
||||
# Validate Hardware Virtualization Support via:
|
||||
|
||||
@@ -1,9 +1,17 @@
|
||||
{
|
||||
config,
|
||||
lib,
|
||||
pkgs,
|
||||
nixpkgsConfig,
|
||||
...
|
||||
}:
|
||||
let
|
||||
# Store hygiene below applies to physical hosts only: VM guests build nothing
|
||||
# locally (Colmena pushes their closures) and MicroVM guests share the host's
|
||||
# read-only store. `hardwareTools` is off on every guest.
|
||||
isPhysical = config.modules.hardwareTools.enable;
|
||||
GiB = 1024 * 1024 * 1024;
|
||||
in
|
||||
{
|
||||
# Nix 2.35 is newer than nixpkgs' nixVersions.stable (still 2.34), so opt into
|
||||
# the latest released version until stable catches up.
|
||||
@@ -25,11 +33,26 @@
|
||||
options = lib.mkDefault "--delete-older-than 7d";
|
||||
};
|
||||
|
||||
# A run can fail while a store path is still in use; retry instead of waiting a week.
|
||||
systemd.services.nix-gc.serviceConfig.Restart = lib.mkIf isPhysical "on-failure";
|
||||
|
||||
# Builds cut off by a crash or power loss stay in the build dir (/nix/var/nix/builds,
|
||||
# Nix's default `build-dir`) on the persistent pool. Clear it at boot, before
|
||||
# nix-daemon runs; the line is boot-only, so Nix's own 7-day tmpfiles rule still
|
||||
# ages the directory at runtime.
|
||||
systemd.tmpfiles.rules = lib.mkIf isPhysical [ "D! /nix/var/nix/builds 0755 root root -" ];
|
||||
|
||||
# Manual optimise storage: nix-store --optimise
|
||||
# https://nixos.org/manual/nix/stable/command-ref/conf-file.html#conf-auto-optimise-store
|
||||
nix.settings = {
|
||||
auto-optimise-store = true;
|
||||
|
||||
# When a build leaves less than 20 GiB free, GC until 50 GiB is free. The
|
||||
# store shares its btrfs pool with /persistent (databases, VM images), which
|
||||
# must keep room to write.
|
||||
min-free = lib.mkIf isPhysical (20 * GiB);
|
||||
max-free = lib.mkIf isPhysical (50 * GiB);
|
||||
|
||||
# Required to run nixpkgs' systemd-nspawn based NixOS tests locally (used
|
||||
# by the nixpkgs-review workflow). These are daemon-wide settings, so every
|
||||
# local build pays the small uid/cgroup allocation overhead; drop them if
|
||||
|
||||
Reference in new issue
Block a user