feat(youko): add Immich photo platform (#317)

Add Immich via the native nixpkgs services.immich module on youko, reusing the host PostgreSQL 16 (pgvector + vectorchord) and a shared Valkey instance for Redis-compatible caching and BullMQ queues (Dragonfly rejected BullMQ's dynamic Lua keys). Media lives on the encrypted HDD at /data/apps/immich; Caddy publishes immich.writefor.fun with a publicly-trusted Let's Encrypt cert (security.acme + Cloudflare DNS-01) so the mobile app connects. OTEL metrics are scraped by VictoriaMetrics, and /var/cache is now preserved on the tmpfs-root hosts so the ML model cache survives reboots.
This commit is contained in:
ryan4yin | 二花 authored and GitHub committed 2026-09-25 20:30:01 +08:00
1 parent 6a6950e7eb
commit 793b636fa5
9 files changed
+338 -19

No files matched your search

Generated
+3 -3
View File
@@ -503,10 +503,10 @@
"mysecrets": {
"flake": false,
"locked": {
"lastModified": 1790320138,
"narHash": "sha256-wx4fG9vS1Fde03PKYMyFNR+4PVaS2Hi1TqKnByAWjHQ=",
"lastModified": 1790329591,
"narHash": "sha256-4dvs20OksqY8orv+w7G/nK9Nvz8MA6uPEhMOm4TQgSQ=",
"ref": "refs/heads/main",
"rev": "5063301c99e72244abba99cd5ae8456626ff42e0",
"rev": "b41ddcfa4537cfb56d1e0f0df0198a2e90fee180",
"shallow": true,
"type": "git",
"url": "ssh://git@github.com/ryan4yin/nix-secrets.git"
+1
View File
@@ -35,6 +35,7 @@ in
"/var/log"
"/var/lib"
"/var/cache"
# k3s related
"/etc/iscsi"
@@ -2,11 +2,17 @@
mylib,
pkgs,
config,
myvars,
wallpapers,
...
}:
let
hostCommonConfig = ''
# TLS for LAN-only services, served with the homelab's self-signed `ecc-ca`
# certificate. Only clients that already installed the private CA (this
# repo's desktops/servers, via `modules/base/security.nix`) trust it.
# Mobile apps do NOT trust a private CA, so any site a phone must reach uses
# `publicClientTlsConfig` instead of this one.
privateTlsConfig = ''
encode zstd gzip
tls ${mylib.relativeToRoot "certs/ecc-server.crt"} ${
config.age.secrets."caddy-ecc-server.key".path
@@ -15,6 +21,23 @@ let
curves x25519 secp384r1 secp521r1
}
'';
# TLS for sites that must be reachable from unmodified mobile apps. The cert
# is issued by a public CA (Let's Encrypt) so the phone app trusts it without
# installing `ecc-ca`. Caddy does NOT issue it: auto-HTTPS is off
# (`auto_https disable_certs`) and the private wildcard cert already covers
# every `*.writefor.fun` name, so caddy would refuse to manage this one.
# Instead NixOS `security.acme` (lego) runs the ACME DNS-01 challenge against
# the Cloudflare zone (see the `security.acme` block below) and caddy only
# loads the resulting files. DNS-01 needs no inbound ports or public IP, so
# the service stays on the LAN; the hostname does appear in public CT logs.
publicClientTlsConfig = domain: ''
encode zstd gzip
tls /var/lib/acme/${domain}/fullchain.pem /var/lib/acme/${domain}/key.pem {
protocols tls1.3 tls1.3
curves x25519 secp384r1 secp521r1
}
'';
in
{
services.caddy = {
@@ -34,14 +57,14 @@ in
# Dashboard
virtualHosts."home.writefor.fun".extraConfig = ''
${hostCommonConfig}
${privateTlsConfig}
reverse_proxy http://localhost:54401
'';
# https://caddyserver.com/docs/caddyfile/directives/file_server
virtualHosts."file.writefor.fun".extraConfig = ''
root * /var/lib/caddy/fileserver/
${hostCommonConfig}
${privateTlsConfig}
file_server browse {
hide .git
precompressed zstd br gzip
@@ -49,23 +72,23 @@ in
'';
virtualHosts."git.writefor.fun".extraConfig = ''
${hostCommonConfig}
${privateTlsConfig}
encode zstd gzip
reverse_proxy http://localhost:3301
'';
virtualHosts."sftpgo.writefor.fun".extraConfig = ''
${hostCommonConfig}
${privateTlsConfig}
encode zstd gzip
reverse_proxy http://localhost:3302
'';
virtualHosts."webdav.writefor.fun".extraConfig = ''
${hostCommonConfig}
${privateTlsConfig}
encode zstd gzip
reverse_proxy http://localhost:3303
'';
# the restic REST server the backup clients push to
virtualHosts."restic.writefor.fun".extraConfig = ''
${hostCommonConfig}
${privateTlsConfig}
# metrics are scraped over loopback only, never through this vhost
@metrics path /metrics
respond @metrics 403
@@ -74,7 +97,7 @@ in
# RustFS replaced MinIO. SigV4 signs the Host header, so preserve it. No
# `encode` here: compression would corrupt range/streaming responses.
virtualHosts."s3.writefor.fun".extraConfig = ''
${hostCommonConfig}
${privateTlsConfig}
reverse_proxy http://localhost:9000 {
header_up Host {http.request.host}
transport http {
@@ -89,7 +112,7 @@ in
# (POST / for STS, GET /?x-id=ListBuckets), so only redirect a plain browser
# visit (unsigned GET /) to the UI; signed requests must reach RustFS as-is.
virtualHosts."s3-console.writefor.fun".extraConfig = ''
${hostCommonConfig}
${privateTlsConfig}
@console_root {
method GET
path /
@@ -107,41 +130,52 @@ in
}
}
'';
# Immich photo library (websockets for live updates). Uploads stream through
# Caddy, which has no request-body limit by default.
# Unlike the LAN-only services above, this one is opened from the Immich
# mobile app, which refuses the private `ecc-ca` certificate -- so it gets a
# publicly-trusted Let's Encrypt cert instead (see `publicClientTlsConfig`).
virtualHosts."immich.writefor.fun".extraConfig = ''
${publicClientTlsConfig "immich.writefor.fun"}
reverse_proxy http://localhost:2283 {
header_up Host {http.request.host}
}
'';
# Every other name is not a service on this host: answer 404 instead of
# caddy's empty 200 fallback, which hides typos.
virtualHosts."*.writefor.fun".extraConfig = ''
${hostCommonConfig}
${privateTlsConfig}
respond 404
'';
virtualHosts."transmission.writefor.fun".extraConfig = ''
${hostCommonConfig}
${privateTlsConfig}
encode zstd gzip
reverse_proxy http://localhost:9091
'';
# Monitoring
virtualHosts."uptime-kuma.writefor.fun".extraConfig = ''
${hostCommonConfig}
${privateTlsConfig}
encode zstd gzip
reverse_proxy http://localhost:53350
'';
virtualHosts."grafana.writefor.fun".extraConfig = ''
${hostCommonConfig}
${privateTlsConfig}
encode zstd gzip
reverse_proxy http://localhost:3351
'';
virtualHosts."prometheus.writefor.fun".extraConfig = ''
${hostCommonConfig}
${privateTlsConfig}
encode zstd gzip
reverse_proxy http://localhost:9090
'';
virtualHosts."alertmanager.writefor.fun".extraConfig = ''
${hostCommonConfig}
${privateTlsConfig}
encode zstd gzip
reverse_proxy http://localhost:9093
'';
virtualHosts."vmalert.writefor.fun".extraConfig = ''
${hostCommonConfig}
${privateTlsConfig}
encode zstd gzip
reverse_proxy http://localhost:8880
'';
@@ -151,6 +185,45 @@ in
# reverse_proxy http://localhost:9090
# '';
};
# Issue the certs for `publicClientTlsConfig` sites with lego (packaged in
# nixpkgs, so no custom caddy build is needed) via the ACME DNS-01 challenge
# against the Cloudflare-managed `writefor.fun` zone. The token is an agenix
# environment file holding `CLOUDFLARE_DNS_API_TOKEN=...`.
security.acme = {
acceptTerms = true;
# lego runs as the unprivileged `acme` user by default, which cannot read
# the root-owned agenix file; run it as root so it can.
useRoot = true;
defaults.email = myvars.useremail;
certs."immich.writefor.fun" = {
dnsProvider = "cloudflare";
environmentFile = config.age.secrets."cloudflare-dns-api-token".path;
# lego's own propagation checks are unreliable from youko (its resolver
# never sees the fresh challenge TXT in time), so replace them with a
# fixed wait; the CA still verifies the record from the public internet.
extraLegoRunFlags = [
"--dns.propagation.wait"
"60s"
];
extraLegoRenewFlags = [
"--dns.propagation.wait"
"60s"
];
# let the caddy user read the issued cert/key
group = "caddy";
# caddy keeps running and loads the new cert on renewal
reloadServices = [ "caddy.service" ];
};
};
# caddy loads the lego-issued files at startup, so wait for the acme service;
# otherwise its config fails to load before the cert exists.
systemd.services.caddy = {
wants = [ "acme-immich.writefor.fun.service" ];
after = [ "acme-immich.writefor.fun.service" ];
};
networking.firewall.allowedTCPPorts = [
80
443
@@ -0,0 +1,122 @@
# Immich (photo platform)
Design for the homelab photo platform on youko. Implementation lives in `./default.nix`; this
document is the agreed design and the rationale.
## Goal
- Browse and search the existing 36G phone gallery (`/data/fileshare/mydata/DCIM`) **in place**,
with ML (faces, objects) and duplicate detection.
- Keep the footprint reasonable next to the k3s microVMs on the same 22G host.
## Non-goals (for now)
- Scheduled phone background backup (the mobile app is used for manual uploads).
- Multi-user / family accounts and public share links.
- Off-host/off-site copy — deferred to the planned cloud backup (see Backup).
## Decisions
- **App**: Immich via the nixpkgs `services.immich` module — a native systemd service, **not**
containers.
- **Database**: reuse the host PostgreSQL 16 (`services.postgresql`) instead of a bundled DB
container.
- **Cache**: a single shared **Valkey** instance (`services.redis.package = pkgs.valkey`), reused by
Immich and future services over loopback TCP — not a private per-service Redis, and not Dragonfly
(its Lua handling breaks Immich's BullMQ job queue; see Risks).
- **Existing photos**: exposed as a read-only Immich **External Library**, not imported/copied (no
36G duplication).
- **ML**: enable it, running on **CPU** initially.
## Architecture
- `services.immich` (server) on youko; `host = "127.0.0.1"`, `port = 2283`.
- `services.immich.database.enable = true` and `createDB = true`: the module adds `pgvector` +
`vectorchord` to `services.postgresql`, sets `shared_preload_libraries = [ "vchord.so" ]`, and
creates the `immich` database/user. Confirmed present in this flake's nixpkgs: PG 16.15,
`pgvector` 0.8.6, `vectorchord` 1.1.1, `immich` 3.2.2.
- `services.immich.redis.enable = false` with `host = "127.0.0.1"` and `port = 6379`: Immich uses
the shared Valkey instance (`../valkey.nix`) over loopback TCP; the module then passes
`REDIS_HOSTNAME`/`REDIS_PORT`.
- `services.immich.machine-learning.enable = true` (CPU; `accelerationDevices` left unset — AMD iGPU
ML is unreliable, revisit `/dev/dri/renderD128` later).
- Caddy vhost `immich.writefor.fun` -> `http://127.0.0.1:2283`. Unlike the other vhosts (private
`ecc-ca`), this one is reached by the Immich mobile app, so it gets a publicly-trusted Let's
Encrypt cert via `security.acme` (lego + Cloudflare DNS-01); see `../caddy.nix`.
- A `homepage` entry on the dashboard.
## Storage
- `mediaLocation = "/data/apps/immich"` (the encrypted HDD; ~3.5T free), created by tmpfiles as
`d /data/apps/immich 0750 immich immich`. This holds the managed library and generated
thumbnails/transcodes.
- The Immich database lives in the Postgres data directory (`/var/lib/postgresql/16`), i.e. under
`@persistent`.
## Photo sources and permissions
- The DCIM tree is `0700`, owned by `sftpgo` (`sftpgo:postgres-exporter`). Grant the `immich` user
read access with a recursive **and default** ACL (`u:immich:rX` + `d:u:immich:rX`) so existing and
future files stay readable, without loosening the tree for everyone.
- In the Immich admin UI, add an External Library pointing at the DCIM path and run a scan. Files
stay read-only to Immich; albums/faces metadata is stored in the database.
## Backup and disaster recovery
- **Database**: already covered by the existing youko restic backup, which snapshots
`/btr_pool/@persistent` (Postgres data lives there).
- **Media and existing photos**: both live on the HDD already. A local HDD->HDD backup adds no
protection, so none is added here. This is deferred to the planned cloud copy, which will cover
the photo library.
- **Note for the cloud-copy work**: a raw btrfs snapshot of a running Postgres is only
crash-consistent; prefer a `pg_dump` into the cloud set for a clean restore. That set should
include `mediaLocation` and the external-library source.
- **One-time disruption**: adding the extensions and `shared_preload_libraries` restarts PostgreSQL
(affects the `playground` DB only).
## Monitoring
- Enabled: `IMMICH_API_METRICS_PORT=8081` + `IMMICH_MICROSERVICES_METRICS_PORT=8082` (OTEL metrics,
`IMMICH_TELEMETRY_INCLUDE=all`). VictoriaMetrics scrapes them on this host
(`monitoring/victoriametrics.nix`); caddy does not proxy them.
- Note: Immich has no bind-host option for these listeners, so they bind all interfaces and are
LAN-reachable like `node_exporter:9100`. Add a firewall rule if that must be loopback-only.
## Risks
- Immich's job queue (BullMQ) needs a Redis that tolerates Lua scripts accessing dynamically-built
keys; **Dragonfly rejected these** (`script tried accessing undeclared key`) and every upload
returned 500. We therefore run **Valkey** (`services.redis`), Redis-compatible and the supported
backend.
## Resource budget
- Immich server + ML worker + Redis: roughly 2-4G RAM, alongside the k3s microVMs; youko has ~13G
available.
## Rollout
1. `../valkey.nix`: the shared Valkey instance.
2. `./default.nix`: the `services.immich` configuration (server, database, machine-learning,
tmpfiles).
3. `../caddy.nix`: the `immich.writefor.fun` vhost.
4. `../oci-containers/homepage/config/services.yaml`: dashboard entry.
5. Deploy: `colmena apply dry-activate` then `switch --on '@youko'`.
6. Register the first user (becomes admin), disable open signup, add the external library, trigger a
scan.
## Validation
- `just fmt`; `just test` -> `true`; `nix build` the youko toplevel.
- Postgres: `\dx` lists `vectorchord`/`pgvector`; `SHOW shared_preload_libraries` includes
`vchord.so`; the `immich` DB exists.
- Units: `immich-server`, `immich-machine-learning`, `redis-shared`, `postgresql` are active.
- `curl 127.0.0.1:2283` and `https://immich.writefor.fun` respond; login works; the external library
scan indexes the DCIM photos and ML search returns results.
## Future
- iGPU ML (`/dev/dri/renderD128`) if it proves reliable.
- `immich-public-proxy` / `immich-kiosk` / `immichframe` for sharing and dashboards.
- Mobile apps and phone auto-backup.
- `pg_dump` of the Immich DB into the cloud backup set.
@@ -0,0 +1,58 @@
{
...
}:
let
dataDir = "/data/apps/immich";
in
{
services.immich = {
enable = true;
host = "127.0.0.1";
port = 2283;
mediaLocation = dataDir;
# Reuse the host PostgreSQL 16 (postgresql.nix): the module adds pgvector +
# vectorchord, sets shared_preload_libraries, and creates the immich
# database and user. No separate database container.
database = {
enable = true;
createDB = true;
};
# Use the shared Valkey instance (valkey.nix) instead of a private Redis.
# With a non-unix-socket host the module passes REDIS_HOSTNAME/PORT.
redis = {
enable = false;
host = "127.0.0.1";
port = 6379;
};
machine-learning.enable = true;
settings = {
server.externalDomain = "https://immich.writefor.fun";
# Skip Immich's outbound version checks.
newVersionCheck.enabled = false;
};
# OTEL metrics, scraped by VictoriaMetrics on this host (see
# monitoring/victoriametrics.nix). `all` collects host/api/io/repo/job.
# Immich binds the metrics listeners to all interfaces (no bind-host
# option), so they are LAN-reachable like node_exporter:9100; caddy does
# not proxy them.
environment = {
IMMICH_API_METRICS_PORT = "8081";
IMMICH_MICROSERVICES_METRICS_PORT = "8082";
IMMICH_TELEMETRY_INCLUDE = "all";
};
};
# mediaLocation is on the encrypted HDD and must stay non-world-readable; the
# module only creates it when left at the default path.
systemd.tmpfiles.rules = [
"d ${dataDir} 0750 immich immich -"
];
# The media lives on /data (a separate mount), so order immich after it.
systemd.services.immich-server.unitConfig.RequiresMountsFor = dataDir;
}
@@ -188,6 +188,38 @@ in
}
];
}
{
job_name = "immich-api";
scrape_interval = "30s";
metrics_path = "/metrics";
static_configs = [
{
# same-host exporter bound to all interfaces (127.0.0.1:8081)
targets = [ "127.0.0.1:8081" ];
labels.type = "app";
labels.app = "immich";
labels.host = "youko";
labels.env = "homelab";
labels.cluster = "homelab";
}
];
}
{
job_name = "immich-microservices";
scrape_interval = "30s";
metrics_path = "/metrics";
static_configs = [
{
# same-host exporter bound to all interfaces (127.0.0.1:8082)
targets = [ "127.0.0.1:8082" ];
labels.type = "app";
labels.app = "immich";
labels.host = "youko";
labels.env = "homelab";
labels.cluster = "homelab";
}
];
}
]
# --- Hosts --- #
++ (lib.attrsets.foldlAttrs
@@ -65,6 +65,11 @@
href: "https://transmission.writefor.fun/"
description: Transmission BitTorrent client
siteMonitor: "https://transmission.writefor.fun/"
- Immich:
icon: immich.png
href: "https://immich.writefor.fun/"
description: Self-hosted photo library
siteMonitor: "https://immich.writefor.fun/"
# - Kubernetes Monitoring:
# # TODO: Update this
# - Emby:
@@ -0,0 +1,19 @@
{
pkgs,
...
}:
{
# Shared Redis-compatible cache for the host; Immich points here over loopback.
# Valkey (BSD Redis fork), not Dragonfly: the latter rejects BullMQ's dynamic-key Lua scripts.
# maxmemory + noeviction: bound memory, and fail a full queue's writes instead of dropping jobs.
services.redis.package = pkgs.valkey;
services.redis.servers.shared = {
enable = true;
bind = "127.0.0.1";
port = 6379;
settings = {
maxmemory = "4gb";
"maxmemory-policy" = "noeviction";
};
};
}
+9
View File
@@ -218,6 +218,15 @@ in
mode = "0400";
owner = "postgres";
};
# Cloudflare API token (Zone -> DNS -> Edit on `writefor.fun`) consumed
# by security.acme (lego) as its environment file for the ACME DNS-01
# challenge on publicly-trusted sites. lego runs as root here.
"cloudflare-dns-api-token" = {
file = "${mysecrets}/server/cloudflare-dns-api-token.env.age";
mode = "0400";
owner = "root";
};
};
})