mirror of
https://github.com/ryan4yin/nix-config.git
synced 2026-11-18 22:08:16 +01:00
feat(remote-builders): build ai and shoukei on aquamarine (#467)
Aquamarine builds for ai and shoukei across the three platforms, with both daemons logging in as root. They authenticate with their passphrase-free SSH host keys, and ssh_known_hosts now carries each builder's address. Assisted-by: DeepSeek Harness 0.2.0-rc.2 (deepseek-flash)
This commit is contained in:
1 parent
4587c6bd14
commit
d6450d48a2
12 files changed
+106
-72
No files matched your search
@@ -62,7 +62,6 @@ modules/
|
||||
└── server/ # Server-specific modules
|
||||
├── qemu-guest-hardware-configuration.nix
|
||||
├── qemu-guest.nix # VM guest base module (libvirt/QEMU guests)
|
||||
├── remote-building.nix # Remote build setup
|
||||
├── server-aarch64.nix
|
||||
├── server-riscv64.nix # module only; the riscv64 output is disabled
|
||||
└── server.nix
|
||||
|
||||
@@ -0,0 +1,70 @@
|
||||
{
|
||||
lib,
|
||||
config,
|
||||
myvars,
|
||||
...
|
||||
}:
|
||||
let
|
||||
cfg = config.modules.remote-builders;
|
||||
|
||||
# Aquamarine - the headless AI inference server (hosts/idols-aquamarine/).
|
||||
# 24C/96G and boot.binfmt emulation for aarch64-linux and riscv64-linux, so it
|
||||
# serves all three platforms this flake targets. `aquamarine` is also its SSH
|
||||
# alias in /etc/ssh/ssh_config, and its host key is pinned in
|
||||
# /etc/ssh/ssh_known_hosts (vars/networking.nix).
|
||||
buildMachines = [
|
||||
{
|
||||
hostName = "aquamarine";
|
||||
systems = [
|
||||
"x86_64-linux" # native
|
||||
"aarch64-linux" # qemu-user via binfmt
|
||||
"riscv64-linux" # qemu-user via binfmt
|
||||
];
|
||||
# `kvm` is omitted even though aquamarine virtualizes well: a foreign
|
||||
# derivation requiring it would otherwise be scheduled there and fail under
|
||||
# emulation instead of falling back to local execution.
|
||||
supportedFeatures = [
|
||||
"benchmark"
|
||||
"big-parallel"
|
||||
];
|
||||
# root is the daemon's default trusted user on the builder; a normal user
|
||||
# would have to be made root-equivalent there to serve builds.
|
||||
sshUser = "root";
|
||||
# The daemon's ssh is non-interactive and cannot reach an agent, so the
|
||||
# builder key must be passphrase-free; the agenix key that interactive
|
||||
# logins use is not. Aquamarine authorizes this host key for root
|
||||
# (hosts/idols-aquamarine/default.nix).
|
||||
sshKey = "/etc/ssh/ssh_host_ed25519_key";
|
||||
inherit (cfg) maxJobs speedFactor;
|
||||
protocol = "ssh-ng";
|
||||
}
|
||||
];
|
||||
in
|
||||
{
|
||||
options.modules.remote-builders = {
|
||||
enable = lib.mkEnableOption "building through aquamarine instead of locally";
|
||||
|
||||
# Nix ranks machines by (running jobs + 1) / speedFactor and picks the
|
||||
# lowest. Keeping this below 2 * speedFactor / 3 means the builder is
|
||||
# preferred for every job it has a slot for, while the slot count stays
|
||||
# under its core count and each job may use several cores anyway — part of
|
||||
# the machine is left for what it actually runs (llama-swap). Once the
|
||||
# slots are full, jobs go to the local machine.
|
||||
maxJobs = lib.mkOption {
|
||||
type = lib.types.ints.positive;
|
||||
default = 12;
|
||||
description = "Concurrent builds to run on the builder.";
|
||||
};
|
||||
|
||||
speedFactor = lib.mkOption {
|
||||
type = lib.types.numbers.positive;
|
||||
default = 24;
|
||||
description = "Speed of the builder relative to this machine.";
|
||||
};
|
||||
};
|
||||
|
||||
config = lib.mkIf cfg.enable {
|
||||
nix.distributedBuilds = true;
|
||||
nix.buildMachines = buildMachines;
|
||||
};
|
||||
}
|
||||
@@ -1,65 +0,0 @@
|
||||
{ myvars, ... }:
|
||||
{
|
||||
####################################################################
|
||||
#
|
||||
# NixOS's Configuration for Remote Building / Distributed Building
|
||||
#
|
||||
# Related Docs:
|
||||
# 1. https://github.com/NixOS/nix/issues/7380
|
||||
# 2. https://nixos.wiki/wiki/Distributed_build
|
||||
# 3. https://github.com/NixOS/nix/issues/2589
|
||||
#
|
||||
####################################################################
|
||||
|
||||
# set local's max-job to 0 to force remote building(disable local building)
|
||||
# nix.settings.max-jobs = 0;
|
||||
nix.distributedBuilds = true;
|
||||
nix.buildMachines =
|
||||
let
|
||||
sshUser = myvars.username;
|
||||
# ssh key's path on local machine; a decrypted secret AI agents must not read
|
||||
sshKey = "/etc/agenix/ssh-key-romantic";
|
||||
systems = [
|
||||
# native arch
|
||||
"x86_64-linux"
|
||||
|
||||
# emulated arch using binfmt_misc and qemu-user
|
||||
"aarch64-linux"
|
||||
"riscv64-linux"
|
||||
];
|
||||
# all available system features are poorly documentd here:
|
||||
# https://github.com/NixOS/nix/blob/e503ead/src/libstore/globals.hh#L673-L687
|
||||
supportedFeatures = [
|
||||
"benchmark"
|
||||
"big-parallel"
|
||||
"kvm"
|
||||
];
|
||||
in
|
||||
[
|
||||
# Nix seems always try to build on the machine remotely
|
||||
# to make use of the local machine's high-performance CPU, do not set remote builder's maxJobs too high.
|
||||
# {
|
||||
# # some of my remote builders are running NixOS
|
||||
# # and has the same sshUser, sshKey, systems, etc.
|
||||
# inherit sshUser sshKey systems supportedFeatures;
|
||||
#
|
||||
# # the hostName should be:
|
||||
# # 1. a hostname that can be resolved by DNS
|
||||
# # 2. the ip address of the remote builder
|
||||
# # 3. a host alias defined globally in /etc/ssh/ssh_config
|
||||
# hostName = "ruby";
|
||||
# maxJobs = 2;
|
||||
# speedFactor = 1;
|
||||
# }
|
||||
# {
|
||||
# inherit sshUser sshKey systems supportedFeatures;
|
||||
# hostName = "kana";
|
||||
# maxJobs = 2;
|
||||
# speedFactor = 1;
|
||||
# }
|
||||
];
|
||||
# optional, useful when the builder has a faster internet connection than yours
|
||||
nix.extraOptions = ''
|
||||
builders-use-substitutes = true
|
||||
'';
|
||||
}
|
||||
@@ -1,7 +1,6 @@
|
||||
{ ... }:
|
||||
{
|
||||
imports = [
|
||||
./remote-building.nix
|
||||
../base
|
||||
../../base
|
||||
];
|
||||
|
||||
Reference in new issue
Block a user