From 13d95daa36db59611d211eda413597dea18dd309 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?ryan4yin=20=7C=20=E4=BA=8C=E8=8A=B1?= Date: Wed, 4 Nov 2026 03:28:30 -0600 Subject: [PATCH] fix(remote-builders): serve x86_64 only, drop aquamarine binfmt (#517) Assisted-by: DeepSeek Harness (deepseek-flash) --- docs/nixos-modernization-options.md | 2 +- hosts/idols-aquamarine/README.md | 5 +++-- .../hardware-configuration.nix | 9 --------- modules/nixos/base/remote-builders.nix | 18 +++++++----------- .../tests/remote-builders/expected.nix | 1 + .../tests/remote-builders/expr.nix | 2 ++ .../tests/remote-builders/expected.nix | 6 ++++++ .../tests/remote-builders/expr.nix | 5 +++++ 8 files changed, 25 insertions(+), 23 deletions(-) diff --git a/docs/nixos-modernization-options.md b/docs/nixos-modernization-options.md index e465f94e..c669cae5 100644 --- a/docs/nixos-modernization-options.md +++ b/docs/nixos-modernization-options.md @@ -235,7 +235,7 @@ no-trusted-user policy; `local-overlay-store` can cut store duplication for remo **Cons:** experimental features in a repo whose value is reproducibility; `nix-servd` is young; misconfiguration shows up as flaky builds rather than a clean failure; benefit is unclear while the -current three-arch remote build setup works. +current remote build setup works. **Reversibility:** medium (builds can be re-pointed, but half-migrated builders are confusing). diff --git a/hosts/idols-aquamarine/README.md b/hosts/idols-aquamarine/README.md index e29183a7..de6cd0b4 100644 --- a/hosts/idols-aquamarine/README.md +++ b/hosts/idols-aquamarine/README.md @@ -269,8 +269,9 @@ group is what requires the recipient step above. ## Remote builds -Aquamarine is the fleet's remote Nix builder: 24C/96G, plus `boot.binfmt.emulatedSystems` for -`aarch64-linux` and `riscv64-linux`, so one machine serves all three platforms. Clients opt in with +Aquamarine is the fleet's remote Nix builder: 24C/96G, serving native `x86_64-linux` only. It runs +no `boot.binfmt.emulatedSystems`: emulated builds are ~10x slower per core, shoukei builds aarch64 +natively, and this flake has no riscv64 output. Clients opt in with `modules.remote-builders.enable`, defined in `modules/nixos/base/remote-builders.nix`. The builder's login user is `root`, which the daemon trusts by default, so this host needs no diff --git a/hosts/idols-aquamarine/hardware-configuration.nix b/hosts/idols-aquamarine/hardware-configuration.nix index d0ee6c46..8563c025 100644 --- a/hosts/idols-aquamarine/hardware-configuration.nix +++ b/hosts/idols-aquamarine/hardware-configuration.nix @@ -34,15 +34,6 @@ # clear /tmp on boot to get a stateless /tmp directory. boot.tmp.cleanOnBoot = true; - # Enable binfmt emulation of aarch64-linux, this is required for cross compilation. - boot.binfmt.emulatedSystems = [ - "aarch64-linux" - "riscv64-linux" - ]; - # This enables the kernel to preload the emulator binaries when the binfmt registrations are added, - # obviating the need to make the emulator binaries available inside chroots and chroot-like sandboxes. - boot.binfmt.preferStaticEmulators = true; # required to work with podman - # supported file systems, so we can mount any removable disks with these filesystems boot.supportedFilesystems = [ "ext4" diff --git a/modules/nixos/base/remote-builders.nix b/modules/nixos/base/remote-builders.nix index 38151ef4..0934f1de 100644 --- a/modules/nixos/base/remote-builders.nix +++ b/modules/nixos/base/remote-builders.nix @@ -8,21 +8,17 @@ let cfg = config.modules.remote-builders; # Aquamarine - the headless AI inference server (hosts/idols-aquamarine/). - # 24C/96G and boot.binfmt emulation for aarch64-linux and riscv64-linux, so it - # serves all three platforms this flake targets. `aquamarine` is also its SSH - # alias in /etc/ssh/ssh_config, and its host key is pinned in + # Native x86_64 only: an aarch64 build is faster on the client that needs it + # than under qemu-user here (hosts/idols-aquamarine/README.md), and this flake + # has no riscv64 output. `aquamarine` is also its SSH alias in + # /etc/ssh/ssh_config, and its host key is pinned in # /etc/ssh/ssh_known_hosts (vars/networking.nix). buildMachines = [ { hostName = "aquamarine"; - systems = [ - "x86_64-linux" # native - "aarch64-linux" # qemu-user via binfmt - "riscv64-linux" # qemu-user via binfmt - ]; - # `kvm` is omitted even though aquamarine virtualizes well: a foreign - # derivation requiring it would otherwise be scheduled there and fail under - # emulation instead of falling back to local execution. + systems = [ "x86_64-linux" ]; + # `kvm` stays unadvertised: kvm-only derivations (VM tests) should run on + # the client instead of competing with the inference server. supportedFeatures = [ "benchmark" "big-parallel" diff --git a/outputs/aarch64-linux/tests/remote-builders/expected.nix b/outputs/aarch64-linux/tests/remote-builders/expected.nix index 158b6392..ab6ad1b9 100644 --- a/outputs/aarch64-linux/tests/remote-builders/expected.nix +++ b/outputs/aarch64-linux/tests/remote-builders/expected.nix @@ -1,4 +1,5 @@ { distributedBuilds = true; aquamarineConfigured = true; + builderSystems = [ [ "x86_64-linux" ] ]; } diff --git a/outputs/aarch64-linux/tests/remote-builders/expr.nix b/outputs/aarch64-linux/tests/remote-builders/expr.nix index 4407fd99..a54ac32f 100644 --- a/outputs/aarch64-linux/tests/remote-builders/expr.nix +++ b/outputs/aarch64-linux/tests/remote-builders/expr.nix @@ -11,4 +11,6 @@ in aquamarineConfigured = builtins.any ( machine: machine.hostName == "aquamarine" && machine.sshUser == "root" ) cfg.nix.buildMachines; + # shoukei builds aarch64 natively, so the builder must not claim it. + builderSystems = map (machine: machine.systems) cfg.nix.buildMachines; } diff --git a/outputs/x86_64-linux/tests/remote-builders/expected.nix b/outputs/x86_64-linux/tests/remote-builders/expected.nix index 40b5a912..20facc00 100644 --- a/outputs/x86_64-linux/tests/remote-builders/expected.nix +++ b/outputs/x86_64-linux/tests/remote-builders/expected.nix @@ -13,4 +13,10 @@ in { distributedBuilds = yes; aquamarineConfigured = yes; + builderSystems = lib.genAttrs [ + "ai-niri" + "shoryu" + "shushou" + "youko" + ] (_: [ [ "x86_64-linux" ] ]); } diff --git a/outputs/x86_64-linux/tests/remote-builders/expr.nix b/outputs/x86_64-linux/tests/remote-builders/expr.nix index fab86b74..a7658db1 100644 --- a/outputs/x86_64-linux/tests/remote-builders/expr.nix +++ b/outputs/x86_64-linux/tests/remote-builders/expr.nix @@ -19,4 +19,9 @@ in { distributedBuilds = lib.genAttrs builderHosts (name: (cfg name).nix.distributedBuilds); aquamarineConfigured = lib.genAttrs builderHosts usesAquamarine; + # The builder must advertise only its native platform: an aarch64 build there + # runs under qemu-user and loses to building it on the client. + builderSystems = lib.genAttrs builderHosts ( + name: map (machine: machine.systems) (cfg name).nix.buildMachines + ); }