feat(remote-builders): build ai and shoukei on aquamarine (#467)

Aquamarine builds for ai and shoukei across the three platforms, with both
daemons logging in as root. They authenticate with their passphrase-free SSH
host keys, and ssh_known_hosts now carries each builder's address.

Assisted-by: DeepSeek Harness 0.2.0-rc.2 (deepseek-flash)
This commit is contained in:
ryan4yin | 二花 authored and GitHub committed 2026-10-17 00:49:22 +08:00
1 parent 4587c6bd14
commit d6450d48a2
12 files changed
+106 -72

No files matched your search

-1
View File
@@ -62,7 +62,6 @@ modules/
└── server/ # Server-specific modules
├── qemu-guest-hardware-configuration.nix
├── qemu-guest.nix # VM guest base module (libvirt/QEMU guests)
├── remote-building.nix # Remote build setup
├── server-aarch64.nix
├── server-riscv64.nix # module only; the riscv64 output is disabled
└── server.nix
+70
View File
@@ -0,0 +1,70 @@
{
lib,
config,
myvars,
...
}:
let
cfg = config.modules.remote-builders;
# Aquamarine - the headless AI inference server (hosts/idols-aquamarine/).
# 24C/96G and boot.binfmt emulation for aarch64-linux and riscv64-linux, so it
# serves all three platforms this flake targets. `aquamarine` is also its SSH
# alias in /etc/ssh/ssh_config, and its host key is pinned in
# /etc/ssh/ssh_known_hosts (vars/networking.nix).
buildMachines = [
{
hostName = "aquamarine";
systems = [
"x86_64-linux" # native
"aarch64-linux" # qemu-user via binfmt
"riscv64-linux" # qemu-user via binfmt
];
# `kvm` is omitted even though aquamarine virtualizes well: a foreign
# derivation requiring it would otherwise be scheduled there and fail under
# emulation instead of falling back to local execution.
supportedFeatures = [
"benchmark"
"big-parallel"
];
# root is the daemon's default trusted user on the builder; a normal user
# would have to be made root-equivalent there to serve builds.
sshUser = "root";
# The daemon's ssh is non-interactive and cannot reach an agent, so the
# builder key must be passphrase-free; the agenix key that interactive
# logins use is not. Aquamarine authorizes this host key for root
# (hosts/idols-aquamarine/default.nix).
sshKey = "/etc/ssh/ssh_host_ed25519_key";
inherit (cfg) maxJobs speedFactor;
protocol = "ssh-ng";
}
];
in
{
options.modules.remote-builders = {
enable = lib.mkEnableOption "building through aquamarine instead of locally";
# Nix ranks machines by (running jobs + 1) / speedFactor and picks the
# lowest. Keeping this below 2 * speedFactor / 3 means the builder is
# preferred for every job it has a slot for, while the slot count stays
# under its core count and each job may use several cores anyway — part of
# the machine is left for what it actually runs (llama-swap). Once the
# slots are full, jobs go to the local machine.
maxJobs = lib.mkOption {
type = lib.types.ints.positive;
default = 12;
description = "Concurrent builds to run on the builder.";
};
speedFactor = lib.mkOption {
type = lib.types.numbers.positive;
default = 24;
description = "Speed of the builder relative to this machine.";
};
};
config = lib.mkIf cfg.enable {
nix.distributedBuilds = true;
nix.buildMachines = buildMachines;
};
}
-65
View File
@@ -1,65 +0,0 @@
{ myvars, ... }:
{
####################################################################
#
# NixOS's Configuration for Remote Building / Distributed Building
#
# Related Docs:
# 1. https://github.com/NixOS/nix/issues/7380
# 2. https://nixos.wiki/wiki/Distributed_build
# 3. https://github.com/NixOS/nix/issues/2589
#
####################################################################
# set local's max-job to 0 to force remote building(disable local building)
# nix.settings.max-jobs = 0;
nix.distributedBuilds = true;
nix.buildMachines =
let
sshUser = myvars.username;
# ssh key's path on local machine; a decrypted secret AI agents must not read
sshKey = "/etc/agenix/ssh-key-romantic";
systems = [
# native arch
"x86_64-linux"
# emulated arch using binfmt_misc and qemu-user
"aarch64-linux"
"riscv64-linux"
];
# all available system features are poorly documentd here:
# https://github.com/NixOS/nix/blob/e503ead/src/libstore/globals.hh#L673-L687
supportedFeatures = [
"benchmark"
"big-parallel"
"kvm"
];
in
[
# Nix seems always try to build on the machine remotely
# to make use of the local machine's high-performance CPU, do not set remote builder's maxJobs too high.
# {
# # some of my remote builders are running NixOS
# # and has the same sshUser, sshKey, systems, etc.
# inherit sshUser sshKey systems supportedFeatures;
#
# # the hostName should be:
# # 1. a hostname that can be resolved by DNS
# # 2. the ip address of the remote builder
# # 3. a host alias defined globally in /etc/ssh/ssh_config
# hostName = "ruby";
# maxJobs = 2;
# speedFactor = 1;
# }
# {
# inherit sshUser sshKey systems supportedFeatures;
# hostName = "kana";
# maxJobs = 2;
# speedFactor = 1;
# }
];
# optional, useful when the builder has a faster internet connection than yours
nix.extraOptions = ''
builders-use-substitutes = true
'';
}
-1
View File
@@ -1,7 +1,6 @@
{ ... }:
{
imports = [
./remote-building.nix
../base
../../base
];