From 793b636fa57542419daee43415f2a4f8c05b4960 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?ryan4yin=20=7C=20=E4=BA=8C=E8=8A=B1?= Date: Fri, 25 Sep 2026 20:30:01 +0800 Subject: [PATCH] feat(youko): add Immich photo platform (#317) Add Immich via the native nixpkgs services.immich module on youko, reusing the host PostgreSQL 16 (pgvector + vectorchord) and a shared Valkey instance for Redis-compatible caching and BullMQ queues (Dragonfly rejected BullMQ's dynamic Lua keys). Media lives on the encrypted HDD at /data/apps/immich; Caddy publishes immich.writefor.fun with a publicly-trusted Let's Encrypt cert (security.acme + Cloudflare DNS-01) so the mobile app connects. OTEL metrics are scraped by VictoriaMetrics, and /var/cache is now preserved on the tmpfs-root hosts so the ML model cache survives reboots. --- flake.lock | 6 +- hosts/12kingdoms-shoryu/preservation.nix | 1 + .../homelab-services/caddy.nix | 105 ++++++++++++--- .../homelab-services/immich/README.md | 122 ++++++++++++++++++ .../homelab-services/immich/default.nix | 58 +++++++++ .../monitoring/victoriametrics.nix | 32 +++++ .../homepage/config/services.yaml | 5 + .../homelab-services/valkey.nix | 19 +++ secrets/nixos.nix | 9 ++ 9 files changed, 338 insertions(+), 19 deletions(-) create mode 100644 hosts/12kingdoms-youko/homelab-services/immich/README.md create mode 100644 hosts/12kingdoms-youko/homelab-services/immich/default.nix create mode 100644 hosts/12kingdoms-youko/homelab-services/valkey.nix diff --git a/flake.lock b/flake.lock index af7cb5d6..e82efce9 100644 --- a/flake.lock +++ b/flake.lock @@ -503,10 +503,10 @@ "mysecrets": { "flake": false, "locked": { - "lastModified": 1790320138, - "narHash": "sha256-wx4fG9vS1Fde03PKYMyFNR+4PVaS2Hi1TqKnByAWjHQ=", + "lastModified": 1790329591, + "narHash": "sha256-4dvs20OksqY8orv+w7G/nK9Nvz8MA6uPEhMOm4TQgSQ=", "ref": "refs/heads/main", - "rev": "5063301c99e72244abba99cd5ae8456626ff42e0", + "rev": "b41ddcfa4537cfb56d1e0f0df0198a2e90fee180", "shallow": true, "type": "git", "url": "ssh://git@github.com/ryan4yin/nix-secrets.git" diff --git a/hosts/12kingdoms-shoryu/preservation.nix b/hosts/12kingdoms-shoryu/preservation.nix index b8b13a1c..d01ff513 100644 --- a/hosts/12kingdoms-shoryu/preservation.nix +++ b/hosts/12kingdoms-shoryu/preservation.nix @@ -35,6 +35,7 @@ in "/var/log" "/var/lib" + "/var/cache" # k3s related "/etc/iscsi" diff --git a/hosts/12kingdoms-youko/homelab-services/caddy.nix b/hosts/12kingdoms-youko/homelab-services/caddy.nix index f76b0abc..9beb5210 100644 --- a/hosts/12kingdoms-youko/homelab-services/caddy.nix +++ b/hosts/12kingdoms-youko/homelab-services/caddy.nix @@ -2,11 +2,17 @@ mylib, pkgs, config, + myvars, wallpapers, ... }: let - hostCommonConfig = '' + # TLS for LAN-only services, served with the homelab's self-signed `ecc-ca` + # certificate. Only clients that already installed the private CA (this + # repo's desktops/servers, via `modules/base/security.nix`) trust it. + # Mobile apps do NOT trust a private CA, so any site a phone must reach uses + # `publicClientTlsConfig` instead of this one. + privateTlsConfig = '' encode zstd gzip tls ${mylib.relativeToRoot "certs/ecc-server.crt"} ${ config.age.secrets."caddy-ecc-server.key".path @@ -15,6 +21,23 @@ let curves x25519 secp384r1 secp521r1 } ''; + + # TLS for sites that must be reachable from unmodified mobile apps. The cert + # is issued by a public CA (Let's Encrypt) so the phone app trusts it without + # installing `ecc-ca`. Caddy does NOT issue it: auto-HTTPS is off + # (`auto_https disable_certs`) and the private wildcard cert already covers + # every `*.writefor.fun` name, so caddy would refuse to manage this one. + # Instead NixOS `security.acme` (lego) runs the ACME DNS-01 challenge against + # the Cloudflare zone (see the `security.acme` block below) and caddy only + # loads the resulting files. DNS-01 needs no inbound ports or public IP, so + # the service stays on the LAN; the hostname does appear in public CT logs. + publicClientTlsConfig = domain: '' + encode zstd gzip + tls /var/lib/acme/${domain}/fullchain.pem /var/lib/acme/${domain}/key.pem { + protocols tls1.3 tls1.3 + curves x25519 secp384r1 secp521r1 + } + ''; in { services.caddy = { @@ -34,14 +57,14 @@ in # Dashboard virtualHosts."home.writefor.fun".extraConfig = '' - ${hostCommonConfig} + ${privateTlsConfig} reverse_proxy http://localhost:54401 ''; # https://caddyserver.com/docs/caddyfile/directives/file_server virtualHosts."file.writefor.fun".extraConfig = '' root * /var/lib/caddy/fileserver/ - ${hostCommonConfig} + ${privateTlsConfig} file_server browse { hide .git precompressed zstd br gzip @@ -49,23 +72,23 @@ in ''; virtualHosts."git.writefor.fun".extraConfig = '' - ${hostCommonConfig} + ${privateTlsConfig} encode zstd gzip reverse_proxy http://localhost:3301 ''; virtualHosts."sftpgo.writefor.fun".extraConfig = '' - ${hostCommonConfig} + ${privateTlsConfig} encode zstd gzip reverse_proxy http://localhost:3302 ''; virtualHosts."webdav.writefor.fun".extraConfig = '' - ${hostCommonConfig} + ${privateTlsConfig} encode zstd gzip reverse_proxy http://localhost:3303 ''; # the restic REST server the backup clients push to virtualHosts."restic.writefor.fun".extraConfig = '' - ${hostCommonConfig} + ${privateTlsConfig} # metrics are scraped over loopback only, never through this vhost @metrics path /metrics respond @metrics 403 @@ -74,7 +97,7 @@ in # RustFS replaced MinIO. SigV4 signs the Host header, so preserve it. No # `encode` here: compression would corrupt range/streaming responses. virtualHosts."s3.writefor.fun".extraConfig = '' - ${hostCommonConfig} + ${privateTlsConfig} reverse_proxy http://localhost:9000 { header_up Host {http.request.host} transport http { @@ -89,7 +112,7 @@ in # (POST / for STS, GET /?x-id=ListBuckets), so only redirect a plain browser # visit (unsigned GET /) to the UI; signed requests must reach RustFS as-is. virtualHosts."s3-console.writefor.fun".extraConfig = '' - ${hostCommonConfig} + ${privateTlsConfig} @console_root { method GET path / @@ -107,41 +130,52 @@ in } } ''; + # Immich photo library (websockets for live updates). Uploads stream through + # Caddy, which has no request-body limit by default. + # Unlike the LAN-only services above, this one is opened from the Immich + # mobile app, which refuses the private `ecc-ca` certificate -- so it gets a + # publicly-trusted Let's Encrypt cert instead (see `publicClientTlsConfig`). + virtualHosts."immich.writefor.fun".extraConfig = '' + ${publicClientTlsConfig "immich.writefor.fun"} + reverse_proxy http://localhost:2283 { + header_up Host {http.request.host} + } + ''; # Every other name is not a service on this host: answer 404 instead of # caddy's empty 200 fallback, which hides typos. virtualHosts."*.writefor.fun".extraConfig = '' - ${hostCommonConfig} + ${privateTlsConfig} respond 404 ''; virtualHosts."transmission.writefor.fun".extraConfig = '' - ${hostCommonConfig} + ${privateTlsConfig} encode zstd gzip reverse_proxy http://localhost:9091 ''; # Monitoring virtualHosts."uptime-kuma.writefor.fun".extraConfig = '' - ${hostCommonConfig} + ${privateTlsConfig} encode zstd gzip reverse_proxy http://localhost:53350 ''; virtualHosts."grafana.writefor.fun".extraConfig = '' - ${hostCommonConfig} + ${privateTlsConfig} encode zstd gzip reverse_proxy http://localhost:3351 ''; virtualHosts."prometheus.writefor.fun".extraConfig = '' - ${hostCommonConfig} + ${privateTlsConfig} encode zstd gzip reverse_proxy http://localhost:9090 ''; virtualHosts."alertmanager.writefor.fun".extraConfig = '' - ${hostCommonConfig} + ${privateTlsConfig} encode zstd gzip reverse_proxy http://localhost:9093 ''; virtualHosts."vmalert.writefor.fun".extraConfig = '' - ${hostCommonConfig} + ${privateTlsConfig} encode zstd gzip reverse_proxy http://localhost:8880 ''; @@ -151,6 +185,45 @@ in # reverse_proxy http://localhost:9090 # ''; }; + + # Issue the certs for `publicClientTlsConfig` sites with lego (packaged in + # nixpkgs, so no custom caddy build is needed) via the ACME DNS-01 challenge + # against the Cloudflare-managed `writefor.fun` zone. The token is an agenix + # environment file holding `CLOUDFLARE_DNS_API_TOKEN=...`. + security.acme = { + acceptTerms = true; + # lego runs as the unprivileged `acme` user by default, which cannot read + # the root-owned agenix file; run it as root so it can. + useRoot = true; + defaults.email = myvars.useremail; + certs."immich.writefor.fun" = { + dnsProvider = "cloudflare"; + environmentFile = config.age.secrets."cloudflare-dns-api-token".path; + # lego's own propagation checks are unreliable from youko (its resolver + # never sees the fresh challenge TXT in time), so replace them with a + # fixed wait; the CA still verifies the record from the public internet. + extraLegoRunFlags = [ + "--dns.propagation.wait" + "60s" + ]; + extraLegoRenewFlags = [ + "--dns.propagation.wait" + "60s" + ]; + # let the caddy user read the issued cert/key + group = "caddy"; + # caddy keeps running and loads the new cert on renewal + reloadServices = [ "caddy.service" ]; + }; + }; + + # caddy loads the lego-issued files at startup, so wait for the acme service; + # otherwise its config fails to load before the cert exists. + systemd.services.caddy = { + wants = [ "acme-immich.writefor.fun.service" ]; + after = [ "acme-immich.writefor.fun.service" ]; + }; + networking.firewall.allowedTCPPorts = [ 80 443 diff --git a/hosts/12kingdoms-youko/homelab-services/immich/README.md b/hosts/12kingdoms-youko/homelab-services/immich/README.md new file mode 100644 index 00000000..1ddf25dc --- /dev/null +++ b/hosts/12kingdoms-youko/homelab-services/immich/README.md @@ -0,0 +1,122 @@ +# Immich (photo platform) + +Design for the homelab photo platform on youko. Implementation lives in `./default.nix`; this +document is the agreed design and the rationale. + +## Goal + +- Browse and search the existing 36G phone gallery (`/data/fileshare/mydata/DCIM`) **in place**, + with ML (faces, objects) and duplicate detection. +- Keep the footprint reasonable next to the k3s microVMs on the same 22G host. + +## Non-goals (for now) + +- Scheduled phone background backup (the mobile app is used for manual uploads). +- Multi-user / family accounts and public share links. +- Off-host/off-site copy — deferred to the planned cloud backup (see Backup). + +## Decisions + +- **App**: Immich via the nixpkgs `services.immich` module — a native systemd service, **not** + containers. +- **Database**: reuse the host PostgreSQL 16 (`services.postgresql`) instead of a bundled DB + container. +- **Cache**: a single shared **Valkey** instance (`services.redis.package = pkgs.valkey`), reused by + Immich and future services over loopback TCP — not a private per-service Redis, and not Dragonfly + (its Lua handling breaks Immich's BullMQ job queue; see Risks). +- **Existing photos**: exposed as a read-only Immich **External Library**, not imported/copied (no + 36G duplication). +- **ML**: enable it, running on **CPU** initially. + +## Architecture + +- `services.immich` (server) on youko; `host = "127.0.0.1"`, `port = 2283`. +- `services.immich.database.enable = true` and `createDB = true`: the module adds `pgvector` + + `vectorchord` to `services.postgresql`, sets `shared_preload_libraries = [ "vchord.so" ]`, and + creates the `immich` database/user. Confirmed present in this flake's nixpkgs: PG 16.15, + `pgvector` 0.8.6, `vectorchord` 1.1.1, `immich` 3.2.2. +- `services.immich.redis.enable = false` with `host = "127.0.0.1"` and `port = 6379`: Immich uses + the shared Valkey instance (`../valkey.nix`) over loopback TCP; the module then passes + `REDIS_HOSTNAME`/`REDIS_PORT`. +- `services.immich.machine-learning.enable = true` (CPU; `accelerationDevices` left unset — AMD iGPU + ML is unreliable, revisit `/dev/dri/renderD128` later). +- Caddy vhost `immich.writefor.fun` -> `http://127.0.0.1:2283`. Unlike the other vhosts (private + `ecc-ca`), this one is reached by the Immich mobile app, so it gets a publicly-trusted Let's + Encrypt cert via `security.acme` (lego + Cloudflare DNS-01); see `../caddy.nix`. +- A `homepage` entry on the dashboard. + +## Storage + +- `mediaLocation = "/data/apps/immich"` (the encrypted HDD; ~3.5T free), created by tmpfiles as + `d /data/apps/immich 0750 immich immich`. This holds the managed library and generated + thumbnails/transcodes. +- The Immich database lives in the Postgres data directory (`/var/lib/postgresql/16`), i.e. under + `@persistent`. + +## Photo sources and permissions + +- The DCIM tree is `0700`, owned by `sftpgo` (`sftpgo:postgres-exporter`). Grant the `immich` user + read access with a recursive **and default** ACL (`u:immich:rX` + `d:u:immich:rX`) so existing and + future files stay readable, without loosening the tree for everyone. +- In the Immich admin UI, add an External Library pointing at the DCIM path and run a scan. Files + stay read-only to Immich; albums/faces metadata is stored in the database. + +## Backup and disaster recovery + +- **Database**: already covered by the existing youko restic backup, which snapshots + `/btr_pool/@persistent` (Postgres data lives there). +- **Media and existing photos**: both live on the HDD already. A local HDD->HDD backup adds no + protection, so none is added here. This is deferred to the planned cloud copy, which will cover + the photo library. +- **Note for the cloud-copy work**: a raw btrfs snapshot of a running Postgres is only + crash-consistent; prefer a `pg_dump` into the cloud set for a clean restore. That set should + include `mediaLocation` and the external-library source. +- **One-time disruption**: adding the extensions and `shared_preload_libraries` restarts PostgreSQL + (affects the `playground` DB only). + +## Monitoring + +- Enabled: `IMMICH_API_METRICS_PORT=8081` + `IMMICH_MICROSERVICES_METRICS_PORT=8082` (OTEL metrics, + `IMMICH_TELEMETRY_INCLUDE=all`). VictoriaMetrics scrapes them on this host + (`monitoring/victoriametrics.nix`); caddy does not proxy them. +- Note: Immich has no bind-host option for these listeners, so they bind all interfaces and are + LAN-reachable like `node_exporter:9100`. Add a firewall rule if that must be loopback-only. + +## Risks + +- Immich's job queue (BullMQ) needs a Redis that tolerates Lua scripts accessing dynamically-built + keys; **Dragonfly rejected these** (`script tried accessing undeclared key`) and every upload + returned 500. We therefore run **Valkey** (`services.redis`), Redis-compatible and the supported + backend. + +## Resource budget + +- Immich server + ML worker + Redis: roughly 2-4G RAM, alongside the k3s microVMs; youko has ~13G + available. + +## Rollout + +1. `../valkey.nix`: the shared Valkey instance. +2. `./default.nix`: the `services.immich` configuration (server, database, machine-learning, + tmpfiles). +3. `../caddy.nix`: the `immich.writefor.fun` vhost. +4. `../oci-containers/homepage/config/services.yaml`: dashboard entry. +5. Deploy: `colmena apply dry-activate` then `switch --on '@youko'`. +6. Register the first user (becomes admin), disable open signup, add the external library, trigger a + scan. + +## Validation + +- `just fmt`; `just test` -> `true`; `nix build` the youko toplevel. +- Postgres: `\dx` lists `vectorchord`/`pgvector`; `SHOW shared_preload_libraries` includes + `vchord.so`; the `immich` DB exists. +- Units: `immich-server`, `immich-machine-learning`, `redis-shared`, `postgresql` are active. +- `curl 127.0.0.1:2283` and `https://immich.writefor.fun` respond; login works; the external library + scan indexes the DCIM photos and ML search returns results. + +## Future + +- iGPU ML (`/dev/dri/renderD128`) if it proves reliable. +- `immich-public-proxy` / `immich-kiosk` / `immichframe` for sharing and dashboards. +- Mobile apps and phone auto-backup. +- `pg_dump` of the Immich DB into the cloud backup set. diff --git a/hosts/12kingdoms-youko/homelab-services/immich/default.nix b/hosts/12kingdoms-youko/homelab-services/immich/default.nix new file mode 100644 index 00000000..4d4401e1 --- /dev/null +++ b/hosts/12kingdoms-youko/homelab-services/immich/default.nix @@ -0,0 +1,58 @@ +{ + ... +}: +let + dataDir = "/data/apps/immich"; +in +{ + services.immich = { + enable = true; + host = "127.0.0.1"; + port = 2283; + mediaLocation = dataDir; + + # Reuse the host PostgreSQL 16 (postgresql.nix): the module adds pgvector + + # vectorchord, sets shared_preload_libraries, and creates the immich + # database and user. No separate database container. + database = { + enable = true; + createDB = true; + }; + + # Use the shared Valkey instance (valkey.nix) instead of a private Redis. + # With a non-unix-socket host the module passes REDIS_HOSTNAME/PORT. + redis = { + enable = false; + host = "127.0.0.1"; + port = 6379; + }; + + machine-learning.enable = true; + + settings = { + server.externalDomain = "https://immich.writefor.fun"; + # Skip Immich's outbound version checks. + newVersionCheck.enabled = false; + }; + + # OTEL metrics, scraped by VictoriaMetrics on this host (see + # monitoring/victoriametrics.nix). `all` collects host/api/io/repo/job. + # Immich binds the metrics listeners to all interfaces (no bind-host + # option), so they are LAN-reachable like node_exporter:9100; caddy does + # not proxy them. + environment = { + IMMICH_API_METRICS_PORT = "8081"; + IMMICH_MICROSERVICES_METRICS_PORT = "8082"; + IMMICH_TELEMETRY_INCLUDE = "all"; + }; + }; + + # mediaLocation is on the encrypted HDD and must stay non-world-readable; the + # module only creates it when left at the default path. + systemd.tmpfiles.rules = [ + "d ${dataDir} 0750 immich immich -" + ]; + + # The media lives on /data (a separate mount), so order immich after it. + systemd.services.immich-server.unitConfig.RequiresMountsFor = dataDir; +} diff --git a/hosts/12kingdoms-youko/homelab-services/monitoring/victoriametrics.nix b/hosts/12kingdoms-youko/homelab-services/monitoring/victoriametrics.nix index 5bfb747a..1c133ff0 100644 --- a/hosts/12kingdoms-youko/homelab-services/monitoring/victoriametrics.nix +++ b/hosts/12kingdoms-youko/homelab-services/monitoring/victoriametrics.nix @@ -188,6 +188,38 @@ in } ]; } + { + job_name = "immich-api"; + scrape_interval = "30s"; + metrics_path = "/metrics"; + static_configs = [ + { + # same-host exporter bound to all interfaces (127.0.0.1:8081) + targets = [ "127.0.0.1:8081" ]; + labels.type = "app"; + labels.app = "immich"; + labels.host = "youko"; + labels.env = "homelab"; + labels.cluster = "homelab"; + } + ]; + } + { + job_name = "immich-microservices"; + scrape_interval = "30s"; + metrics_path = "/metrics"; + static_configs = [ + { + # same-host exporter bound to all interfaces (127.0.0.1:8082) + targets = [ "127.0.0.1:8082" ]; + labels.type = "app"; + labels.app = "immich"; + labels.host = "youko"; + labels.env = "homelab"; + labels.cluster = "homelab"; + } + ]; + } ] # --- Hosts --- # ++ (lib.attrsets.foldlAttrs diff --git a/hosts/12kingdoms-youko/homelab-services/oci-containers/homepage/config/services.yaml b/hosts/12kingdoms-youko/homelab-services/oci-containers/homepage/config/services.yaml index ebbc1404..f062a25a 100644 --- a/hosts/12kingdoms-youko/homelab-services/oci-containers/homepage/config/services.yaml +++ b/hosts/12kingdoms-youko/homelab-services/oci-containers/homepage/config/services.yaml @@ -65,6 +65,11 @@ href: "https://transmission.writefor.fun/" description: Transmission BitTorrent client siteMonitor: "https://transmission.writefor.fun/" + - Immich: + icon: immich.png + href: "https://immich.writefor.fun/" + description: Self-hosted photo library + siteMonitor: "https://immich.writefor.fun/" # - Kubernetes Monitoring: # # TODO: Update this # - Emby: diff --git a/hosts/12kingdoms-youko/homelab-services/valkey.nix b/hosts/12kingdoms-youko/homelab-services/valkey.nix new file mode 100644 index 00000000..c11c7ffd --- /dev/null +++ b/hosts/12kingdoms-youko/homelab-services/valkey.nix @@ -0,0 +1,19 @@ +{ + pkgs, + ... +}: +{ + # Shared Redis-compatible cache for the host; Immich points here over loopback. + # Valkey (BSD Redis fork), not Dragonfly: the latter rejects BullMQ's dynamic-key Lua scripts. + # maxmemory + noeviction: bound memory, and fail a full queue's writes instead of dropping jobs. + services.redis.package = pkgs.valkey; + services.redis.servers.shared = { + enable = true; + bind = "127.0.0.1"; + port = 6379; + settings = { + maxmemory = "4gb"; + "maxmemory-policy" = "noeviction"; + }; + }; +} diff --git a/secrets/nixos.nix b/secrets/nixos.nix index 20497b03..03b916c4 100644 --- a/secrets/nixos.nix +++ b/secrets/nixos.nix @@ -218,6 +218,15 @@ in mode = "0400"; owner = "postgres"; }; + + # Cloudflare API token (Zone -> DNS -> Edit on `writefor.fun`) consumed + # by security.acme (lego) as its environment file for the ACME DNS-01 + # challenge on publicly-trusted sites. lego runs as root here. + "cloudflare-dns-api-token" = { + file = "${mysecrets}/server/cloudflare-dns-api-token.env.age"; + mode = "0400"; + owner = "root"; + }; }; })