From d6450d48a23e4e11e09f75097c5caa6eee1df6c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?ryan4yin=20=7C=20=E4=BA=8C=E8=8A=B1?= Date: Sat, 17 Oct 2026 00:49:22 +0800 Subject: [PATCH] feat(remote-builders): build ai and shoukei on aquamarine (#467) Aquamarine builds for ai and shoukei across the three platforms, with both daemons logging in as root. They authenticate with their passphrase-free SSH host keys, and ssh_known_hosts now carries each builder's address. Assisted-by: DeepSeek Harness 0.2.0-rc.2 (deepseek-flash) --- docs/nixos-modernization-options.md | 7 ++- hosts/12kingdoms-shoukei/default.nix | 4 ++ hosts/idols-ai/default.nix | 3 + hosts/idols-aquamarine/README.md | 11 ++++ hosts/idols-aquamarine/default.nix | 3 + modules/README.md | 1 - modules/nixos/base/remote-builders.nix | 70 ++++++++++++++++++++++++ modules/nixos/server/remote-building.nix | 65 ---------------------- modules/nixos/server/server.nix | 1 - vars/README.md | 2 +- vars/default.nix | 6 ++ vars/networking.nix | 5 +- 12 files changed, 106 insertions(+), 72 deletions(-) create mode 100644 modules/nixos/base/remote-builders.nix delete mode 100644 modules/nixos/server/remote-building.nix diff --git a/docs/nixos-modernization-options.md b/docs/nixos-modernization-options.md index e7b0f320..253a5c4b 100644 --- a/docs/nixos-modernization-options.md +++ b/docs/nixos-modernization-options.md @@ -202,8 +202,9 @@ cannot go to a public cache. ## A8. nix-servd and the non-root daemon features -**Current:** static `nix.buildMachines` (`modules/nixos/server/remote-building.nix`); plain-user is -deliberately not a trusted user; Nix 2.35 via `nixVersions.latest` (WA-023). +**Current:** static `nix.buildMachines` (`modules/nixos/base/remote-builders.nix`); the plain user +is deliberately not a trusted user on the clients, though the builder host trusts its builder login; +Nix 2.35 via `nixVersions.latest` (WA-023). **Proposed:** `nix-servd` for scheduling, plus `auto-allocate-uids` and `local-overlay-store` experimental features. @@ -440,7 +441,7 @@ adopt individually only afterwards.** | --- | ------------------------------------------------------ | -------------------------------------------- | ----------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------------------------- | | D1 | `system.etc.overlay.enable` | /etc generated by the perl activation script | /etc becomes an EROFS + overlay mount; foundation for an immutable /etc | upstream: _"This is currently experimental. Only enable this option if you are confident that you can recover your system if it breaks."_; every /etc writer must be audited | lab only | | D2 | `systemd.sysusers.enable` / `services.userborn.enable` | `update-users-groups.pl` | removes the perl script; prerequisite for an immutable /etc | experimental; uid/gid migration risk | lab only | -| D3 | `system.etc.overlay.mutable = false` | writable /etc | truly immutable /etc | **blocked**: agenix writes `/etc/agenix` (see `remote-building.nix:21`), so secrets must move to /run or to sops-nix first | lab only, after secrets move | +| D3 | `system.etc.overlay.mutable = false` | writable /etc | truly immutable /etc | **blocked**: agenix writes `/etc/agenix` (see the `environment.etc` copies in `secrets/nixos.nix`), so secrets must move to /run or to sops-nix first | lab only, after secrets move | | D4 | `system.nixos-init.enable` | `activationScripts` | bashless init | makes `activationScripts` a silent no-op — btrbk/restic hooks would stop running without an error | lab only | | D5 | `security.pam.settings` | per-service booleans | freeform PAM config | upstream marks it subject to breaking changes; a mistake locks you out | lab only | | D6 | `security.shadow.enable = false` + greetd | shadow suite provides login/su | fewer setuid programs | assertion couples the two; changes VT login semantics | lab only | diff --git a/hosts/12kingdoms-shoukei/default.nix b/hosts/12kingdoms-shoukei/default.nix index c5a3bac2..98db0c1e 100644 --- a/hosts/12kingdoms-shoukei/default.nix +++ b/hosts/12kingdoms-shoukei/default.nix @@ -19,6 +19,10 @@ in ../idols-ai/preservation.nix ]; + # 16G on a laptop: heavy derivations go to aquamarine's 96G instead of + # swapping here. + modules.remote-builders.enable = true; + # disable sunshine for security services.sunshine.enable = lib.mkForce false; diff --git a/hosts/idols-ai/default.nix b/hosts/idols-ai/default.nix index ce75754b..fc6b7460 100644 --- a/hosts/idols-ai/default.nix +++ b/hosts/idols-ai/default.nix @@ -47,6 +47,9 @@ in # disk swapfile instead, created by disko at /swap (see ./disko-fs.nix). The two must move together. modules.zram.enable = false; + # 32G of RAM against aquamarine's 96G, so builds go to the server first. + modules.remote-builders.enable = true; + # zswap: compressed writeback cache in front of the disk swapfile, so cold # anonymous pages stay in RAM instead of hitting the SSD. Safe here: the disk # swapfile remains the real backing store, no zram deadlock structure. diff --git a/hosts/idols-aquamarine/README.md b/hosts/idols-aquamarine/README.md index 7448743d..66606cdd 100644 --- a/hosts/idols-aquamarine/README.md +++ b/hosts/idols-aquamarine/README.md @@ -221,6 +221,17 @@ reinstall or a key rotation: This config enables no `modules.secrets.*` group, so aquamarine decrypts nothing today; adding a group is what requires the recipient step above. +## Remote builds + +Aquamarine is the fleet's remote Nix builder: 24C/96G, plus `boot.binfmt.emulatedSystems` for +`aarch64-linux` and `riscv64-linux`, so one machine serves all three platforms. Clients opt in with +`modules.remote-builders.enable`, defined in `modules/nixos/base/remote-builders.nix`. + +The builder's login user is `root`, which the daemon trusts by default, so this host needs no +`nix.settings.trusted-users` entry of its own. Each client's daemon authenticates with its own SSH +host key, which is passphrase-free and root-readable; `hosts/idols-aquamarine/default.nix` +authorizes those keys for root. + ## Deploy ```bash diff --git a/hosts/idols-aquamarine/default.nix b/hosts/idols-aquamarine/default.nix index fe1962ef..aa635ba3 100644 --- a/hosts/idols-aquamarine/default.nix +++ b/hosts/idols-aquamarine/default.nix @@ -68,6 +68,9 @@ in # so enabled user services can run without an active login session. users.users."${myvars.username}".linger = true; + # The daemons of the hosts that build here (modules/nixos/base/remote-builders.nix). + users.users.root.openssh.authorizedKeys.keys = myvars.remoteBuilderClientKeys; + # systemd-journal - cap disk usage # https://www.freedesktop.org/software/systemd/man/latest/journald.conf.html services.journald.settings.Journal = { diff --git a/modules/README.md b/modules/README.md index 0895930d..07ff44c8 100644 --- a/modules/README.md +++ b/modules/README.md @@ -62,7 +62,6 @@ modules/ └── server/ # Server-specific modules ├── qemu-guest-hardware-configuration.nix ├── qemu-guest.nix # VM guest base module (libvirt/QEMU guests) - ├── remote-building.nix # Remote build setup ├── server-aarch64.nix ├── server-riscv64.nix # module only; the riscv64 output is disabled └── server.nix diff --git a/modules/nixos/base/remote-builders.nix b/modules/nixos/base/remote-builders.nix new file mode 100644 index 00000000..38151ef4 --- /dev/null +++ b/modules/nixos/base/remote-builders.nix @@ -0,0 +1,70 @@ +{ + lib, + config, + myvars, + ... +}: +let + cfg = config.modules.remote-builders; + + # Aquamarine - the headless AI inference server (hosts/idols-aquamarine/). + # 24C/96G and boot.binfmt emulation for aarch64-linux and riscv64-linux, so it + # serves all three platforms this flake targets. `aquamarine` is also its SSH + # alias in /etc/ssh/ssh_config, and its host key is pinned in + # /etc/ssh/ssh_known_hosts (vars/networking.nix). + buildMachines = [ + { + hostName = "aquamarine"; + systems = [ + "x86_64-linux" # native + "aarch64-linux" # qemu-user via binfmt + "riscv64-linux" # qemu-user via binfmt + ]; + # `kvm` is omitted even though aquamarine virtualizes well: a foreign + # derivation requiring it would otherwise be scheduled there and fail under + # emulation instead of falling back to local execution. + supportedFeatures = [ + "benchmark" + "big-parallel" + ]; + # root is the daemon's default trusted user on the builder; a normal user + # would have to be made root-equivalent there to serve builds. + sshUser = "root"; + # The daemon's ssh is non-interactive and cannot reach an agent, so the + # builder key must be passphrase-free; the agenix key that interactive + # logins use is not. Aquamarine authorizes this host key for root + # (hosts/idols-aquamarine/default.nix). + sshKey = "/etc/ssh/ssh_host_ed25519_key"; + inherit (cfg) maxJobs speedFactor; + protocol = "ssh-ng"; + } + ]; +in +{ + options.modules.remote-builders = { + enable = lib.mkEnableOption "building through aquamarine instead of locally"; + + # Nix ranks machines by (running jobs + 1) / speedFactor and picks the + # lowest. Keeping this below 2 * speedFactor / 3 means the builder is + # preferred for every job it has a slot for, while the slot count stays + # under its core count and each job may use several cores anyway — part of + # the machine is left for what it actually runs (llama-swap). Once the + # slots are full, jobs go to the local machine. + maxJobs = lib.mkOption { + type = lib.types.ints.positive; + default = 12; + description = "Concurrent builds to run on the builder."; + }; + + speedFactor = lib.mkOption { + type = lib.types.numbers.positive; + default = 24; + description = "Speed of the builder relative to this machine."; + }; + }; + + config = lib.mkIf cfg.enable { + nix.distributedBuilds = true; + nix.buildMachines = buildMachines; + }; +} diff --git a/modules/nixos/server/remote-building.nix b/modules/nixos/server/remote-building.nix deleted file mode 100644 index 6726d0b1..00000000 --- a/modules/nixos/server/remote-building.nix +++ /dev/null @@ -1,65 +0,0 @@ -{ myvars, ... }: -{ - #################################################################### - # - # NixOS's Configuration for Remote Building / Distributed Building - # - # Related Docs: - # 1. https://github.com/NixOS/nix/issues/7380 - # 2. https://nixos.wiki/wiki/Distributed_build - # 3. https://github.com/NixOS/nix/issues/2589 - # - #################################################################### - - # set local's max-job to 0 to force remote building(disable local building) - # nix.settings.max-jobs = 0; - nix.distributedBuilds = true; - nix.buildMachines = - let - sshUser = myvars.username; - # ssh key's path on local machine; a decrypted secret AI agents must not read - sshKey = "/etc/agenix/ssh-key-romantic"; - systems = [ - # native arch - "x86_64-linux" - - # emulated arch using binfmt_misc and qemu-user - "aarch64-linux" - "riscv64-linux" - ]; - # all available system features are poorly documentd here: - # https://github.com/NixOS/nix/blob/e503ead/src/libstore/globals.hh#L673-L687 - supportedFeatures = [ - "benchmark" - "big-parallel" - "kvm" - ]; - in - [ - # Nix seems always try to build on the machine remotely - # to make use of the local machine's high-performance CPU, do not set remote builder's maxJobs too high. - # { - # # some of my remote builders are running NixOS - # # and has the same sshUser, sshKey, systems, etc. - # inherit sshUser sshKey systems supportedFeatures; - # - # # the hostName should be: - # # 1. a hostname that can be resolved by DNS - # # 2. the ip address of the remote builder - # # 3. a host alias defined globally in /etc/ssh/ssh_config - # hostName = "ruby"; - # maxJobs = 2; - # speedFactor = 1; - # } - # { - # inherit sshUser sshKey systems supportedFeatures; - # hostName = "kana"; - # maxJobs = 2; - # speedFactor = 1; - # } - ]; - # optional, useful when the builder has a faster internet connection than yours - nix.extraOptions = '' - builders-use-substitutes = true - ''; -} diff --git a/modules/nixos/server/server.nix b/modules/nixos/server/server.nix index b108fbce..c9434748 100644 --- a/modules/nixos/server/server.nix +++ b/modules/nixos/server/server.nix @@ -1,7 +1,6 @@ { ... }: { imports = [ - ./remote-building.nix ../base ../../base ]; diff --git a/vars/README.md b/vars/README.md index 8694cddc..bf7b1377 100644 --- a/vars/README.md +++ b/vars/README.md @@ -30,7 +30,7 @@ Defines: - **DNS servers**: IPv4 and IPv6 name servers - **Host inventory**: static LAN hosts and their interfaces; DHCP/mobile hosts (such as the macOS hosts) are omitted by design -- **SSH configuration**: Remote builder aliases and known hosts configuration +- **SSH configuration**: Host aliases and known hosts configuration - **Network topology**: Physical machines, VMs, Kubernetes clusters, and SBCs ## Host Categories diff --git a/vars/default.nix b/vars/default.nix index 7b04f11e..491ea505 100644 --- a/vars/default.nix +++ b/vars/default.nix @@ -32,4 +32,10 @@ # the backup ssh keys for disaster recovery "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIMzYT0Fpcp681eHY5FJV2G8Mve53iX3hMOLGbVvfL+TF ryan@romantic" ]; + # The SSH host keys of the machines whose nix-daemons build through aquamarine + # (modules/nixos/base/remote-builders.nix). + remoteBuilderClientKeys = [ + "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAINHZtzeaQyXwuRMLzoOAuTu8P9bu5yc5MBwo5LI3iWBV root@ai" + "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAILDkbOwOeuAem8BKbyTDliRiTfHVAFdiVpAKh1bhs2s0 root@shoukei" + ]; } diff --git a/vars/networking.nix b/vars/networking.nix index 15549e81..5c5fbcd4 100644 --- a/vars/networking.nix +++ b/vars/networking.nix @@ -199,7 +199,10 @@ rec { # => { x = "bar-a"; y = "bar-b"; } lib.attrsets.mapAttrs (host: value: { - hostNames = [ host ] ++ (lib.optional (hostsAddr ? host) hostsAddr.${host}.ipv4); + # Name plus address: the Nix daemon connects the builder by its + # address, and verifies the host key under that name. A name-only + # entry fails the build with "Host key verification failed". + hostNames = [ host ] ++ lib.optionals (builtins.hasAttr host hostsAddr) [ hostsAddr.${host}.ipv4 ]; publicKey = value.publicKey; }) {