mirror of
https://github.com/ryan4yin/nix-config.git
synced 2026-11-13 12:13:36 +01:00
Ryan Yin
9b68cfdb8a
The backup unit publishes `restic_backup_last_success_timestamp_seconds` for the node_exporter textfile collector, and a rejected alert now fails the notify unit instead of exiting 0. Assisted-by: DeepSeek Harness 0.2.1-rc.2 (deepseek-flash)
254 lines
9.4 KiB
Nix
254 lines
9.4 KiB
Nix
{
|
|
config,
|
|
lib,
|
|
pkgs,
|
|
...
|
|
}:
|
|
let
|
|
cfg = config.modules.restic-backup;
|
|
|
|
# A short-lived, read-only btrfs snapshot taken by the backup itself, so the
|
|
# backup is a consistent point-in-time (a live copy of, say, a running
|
|
# postgres data directory is only crash-consistent). It is deleted right
|
|
# after, so it is not a second backup - just a consistency step.
|
|
snapshotPath = "${builtins.dirOf cfg.snapshotSource}/@restic-snapshot";
|
|
|
|
# node_exporter's textfile collector directory, kept in sync with
|
|
# modules/nixos/base/monitoring.nix, which points the collector here.
|
|
textfileDir = "/var/lib/node_exporter/textfile_collector";
|
|
in
|
|
{
|
|
# ==================================================================
|
|
#
|
|
# A restic backup to a LOCAL repository (part 1 of the backup plan).
|
|
#
|
|
# Part 2 (copying the critical snapshots to a cloud object store with
|
|
# `restic copy`) is a later step. btrbk keeps making the cheap local btrfs
|
|
# snapshots for rollbacks; off-host copies are restic's job (restic can
|
|
# exclude the regenerable bulk at the file level).
|
|
#
|
|
# Keys and credentials are NEVER included: restic's own password lives in
|
|
# /etc/agenix, so backing that up would store the repository's password
|
|
# inside the repository.
|
|
#
|
|
# ==================================================================
|
|
options.modules.restic-backup = {
|
|
enable = lib.mkEnableOption "a restic backup (local repository)";
|
|
|
|
repository = lib.mkOption {
|
|
type = lib.types.str;
|
|
example = "/data/backups/restic/youko";
|
|
description = "Restic repository, e.g. a path under a mounted backup disk.";
|
|
};
|
|
|
|
snapshotSource = lib.mkOption {
|
|
type = lib.types.nullOr lib.types.str;
|
|
default = null;
|
|
example = "/btr_pool/@persistent";
|
|
description = ''
|
|
btrfs subvolume to back up through a short-lived read-only snapshot.
|
|
When null, `paths` is backed up directly.
|
|
'';
|
|
};
|
|
|
|
paths = lib.mkOption {
|
|
type = lib.types.listOf lib.types.str;
|
|
default = [ ];
|
|
description = "Paths to back up (used when {option}`snapshotSource` is null).";
|
|
};
|
|
|
|
exclude = lib.mkOption {
|
|
type = lib.types.listOf lib.types.str;
|
|
default = [ ];
|
|
description = "Extra exclude patterns, on top of the built-in ones.";
|
|
};
|
|
|
|
requiresMountsFor = lib.mkOption {
|
|
type = lib.types.nullOr lib.types.str;
|
|
default = null;
|
|
description = ''
|
|
Mount point the repository lives on, when it is a `nofail` mount: the
|
|
backup is then ordered after that mount instead of running against a
|
|
not-yet-mounted directory.
|
|
'';
|
|
};
|
|
|
|
excludeLargerThan = lib.mkOption {
|
|
type = lib.types.nullOr lib.types.str;
|
|
default = null;
|
|
example = "500M";
|
|
description = ''
|
|
Skip any file larger than this (restic's `--exclude-larger-than`). A size
|
|
cap is more reliable than guessing file types when the tree contains
|
|
large re-downloadable artifacts.
|
|
'';
|
|
};
|
|
|
|
environmentFile = lib.mkOption {
|
|
type = lib.types.nullOr lib.types.path;
|
|
default = null;
|
|
example = config.age.secrets."restic-rest-credentials".path;
|
|
description = ''
|
|
systemd `EnvironmentFile` with restic environment variables. Needed for
|
|
a `rest:` repository, whose credentials are environment-only
|
|
(`RESTIC_REST_USERNAME` / `RESTIC_REST_PASSWORD`) and must not appear in
|
|
the repository URL.
|
|
'';
|
|
};
|
|
|
|
onCalendar = lib.mkOption {
|
|
type = lib.types.str;
|
|
default = "01:30";
|
|
description = ''
|
|
systemd calendar expression for the backup timer. Give every host its
|
|
own slot rather than relying on the random delay: a client's off-host
|
|
push, the backup server's own run and btrbk (Tue/Sat 03:45:20 by
|
|
default) all contend for the same disks otherwise.
|
|
'';
|
|
};
|
|
|
|
randomizedDelaySec = lib.mkOption {
|
|
type = lib.types.str;
|
|
default = "15m";
|
|
description = "Extra random delay on top of {option}`onCalendar`, to smooth load.";
|
|
};
|
|
|
|
alertManagerUrl = lib.mkOption {
|
|
type = lib.types.str;
|
|
default = "https://alertmanager.writefor.fun";
|
|
description = ''
|
|
Alertmanager to POST a failure alert to (`/api/v2/alerts`). youko routes
|
|
anything above info severity to telegram, so a failed backup is not
|
|
silent.
|
|
'';
|
|
};
|
|
|
|
pruneOpts = lib.mkOption {
|
|
type = lib.types.listOf lib.types.str;
|
|
default = [
|
|
"--keep-daily 3"
|
|
"--keep-weekly 2"
|
|
"--keep-monthly 2"
|
|
];
|
|
description = ''
|
|
Retention passed to `restic forget --prune`. Leave it empty when the
|
|
repository is an append-only server: that server rejects deletion, so
|
|
the client cannot forget or prune (the backup would otherwise fail
|
|
after uploading).
|
|
'';
|
|
};
|
|
|
|
checkOpts = lib.mkOption {
|
|
type = lib.types.listOf lib.types.str;
|
|
default = [ "--read-data-subset=1/20" ];
|
|
description = ''
|
|
Arguments for `restic check`, run after each backup. Non-empty enables
|
|
the check. The default reads 5% of the pack data as a fraction rather
|
|
than `5%`, so systemd does not treat the percent as a specifier; set
|
|
to `[ ]` to disable, or `--read-data` for a full read.
|
|
'';
|
|
};
|
|
};
|
|
|
|
config = lib.mkIf cfg.enable {
|
|
services.restic.backups.homelab = {
|
|
inherit (cfg) repository;
|
|
initialize = true;
|
|
# take the path from agenix itself instead of hardcoding its secretsDir
|
|
passwordFile = config.age.secrets."restic-password".path;
|
|
|
|
paths = if cfg.snapshotSource != null then [ snapshotPath ] else cfg.paths;
|
|
|
|
# Relative to the backed-up tree, so they work for both the snapshot and
|
|
# a direct path.
|
|
exclude = [
|
|
# Universal: keys and credentials are never backed up, on any host.
|
|
# (restic's own password lives in /etc/agenix, so backing that up would
|
|
# store the repository's password inside the repository.)
|
|
#
|
|
# Host-specific excludes (regenerable bulk, VM images, ...) belong in
|
|
# the host's own config.
|
|
"etc/agenix"
|
|
"etc/ssh/ssh_host_*"
|
|
"**/.ssh"
|
|
"**/.gnupg"
|
|
"**/.aws"
|
|
"**/.config/gcloud"
|
|
]
|
|
++ cfg.exclude;
|
|
|
|
# Setting timerConfig replaces the module default, so keep its
|
|
# `Persistent`: a host that is off at the slot backs up once it is back.
|
|
timerConfig = {
|
|
OnCalendar = cfg.onCalendar;
|
|
RandomizedDelaySec = cfg.randomizedDelaySec;
|
|
Persistent = true;
|
|
};
|
|
|
|
inherit (cfg) pruneOpts;
|
|
inherit (cfg) checkOpts;
|
|
}
|
|
// lib.optionalAttrs (cfg.excludeLargerThan != null) {
|
|
extraBackupArgs = [
|
|
"--exclude-larger-than"
|
|
cfg.excludeLargerThan
|
|
];
|
|
}
|
|
|
|
// lib.optionalAttrs (cfg.environmentFile != null) {
|
|
inherit (cfg) environmentFile;
|
|
}
|
|
|
|
// lib.optionalAttrs (cfg.snapshotSource != null) {
|
|
backupPrepareCommand = ''
|
|
${pkgs.btrfs-progs}/bin/btrfs subvolume snapshot -r ${cfg.snapshotSource} ${snapshotPath}
|
|
'';
|
|
backupCleanupCommand = ''
|
|
${pkgs.btrfs-progs}/bin/btrfs subvolume delete ${snapshotPath}
|
|
'';
|
|
};
|
|
|
|
systemd.services.restic-backups-homelab.unitConfig =
|
|
lib.optionalAttrs (cfg.requiresMountsFor != null)
|
|
{
|
|
RequiresMountsFor = cfg.requiresMountsFor;
|
|
};
|
|
|
|
# Record when the last fully successful run finished, so staleness is
|
|
# alertable (hosts/12kingdoms-youko/homelab-services/monitoring/alert_rules/
|
|
# restic-backup.yml). ExecStartPost runs only after every ExecStart
|
|
# (backup, forget --prune, check) succeeded, and the rename keeps
|
|
# node_exporter from reading a half-written file.
|
|
systemd.services.restic-backups-homelab.postStart = ''
|
|
${pkgs.coreutils}/bin/install -d -m 0755 ${textfileDir}
|
|
{
|
|
echo '# HELP restic_backup_last_success_timestamp_seconds Unix time of the last successful restic backup run.'
|
|
echo '# TYPE restic_backup_last_success_timestamp_seconds gauge'
|
|
echo "restic_backup_last_success_timestamp_seconds $(${pkgs.coreutils}/bin/date +%s)"
|
|
} > ${textfileDir}/restic-backup.prom.tmp
|
|
${pkgs.coreutils}/bin/mv -f ${textfileDir}/restic-backup.prom.tmp ${textfileDir}/restic-backup.prom
|
|
'';
|
|
|
|
# A failed backup must not be silent: report it to alertmanager, which
|
|
# routes anything above info severity to telegram.
|
|
systemd.services.restic-backups-homelab.onFailure = [ "restic-backup-notify.service" ];
|
|
|
|
systemd.services.restic-backup-notify = {
|
|
description = "Report a failed restic backup";
|
|
serviceConfig.Type = "oneshot";
|
|
# `--fail` makes curl exit non-zero on an HTTP 4xx/5xx response: without
|
|
# it a rejected alert still ended the unit with status 0, i.e. silently.
|
|
# `--max-time` bounds the request so an unresponsive Alertmanager cannot
|
|
# hold the unit open.
|
|
script = ''
|
|
${pkgs.curl}/bin/curl -sS --fail --max-time 10 -o /dev/null -X POST ${cfg.alertManagerUrl}/api/v2/alerts \
|
|
-H 'Content-Type: application/json' \
|
|
--data "[{ \
|
|
\"labels\": { \"alertname\": \"ResticBackupFailed\", \"severity\": \"critical\", \"host\": \"${config.networking.hostName}\" }, \
|
|
\"annotations\": { \"summary\": \"restic backup failed on ${config.networking.hostName}\", \"description\": \"journalctl -u restic-backups-homelab\" } \
|
|
}]"
|
|
'';
|
|
};
|
|
};
|
|
}
|