mirror of
https://github.com/ryan4yin/nix-config.git
synced 2026-11-13 12:13:46 +01:00
feat(aquamarine): revive aquamarine as a two-GPU inference host
The Z890 board and its root SSD move here from `ai`, and the retired VM's name and 192.168.5.101 are reused. Assisted-by: DeepSeek Harness (deepseek-flash)
This commit is contained in:
Ryan Yin authored and ryan4yin | 二花 committed
1 parent
9c84636f10
commit
72a7a2ae88
21 files changed
+1268
-37
No files matched your search
@@ -63,4 +63,8 @@ extend-ignore-re = [
|
||||
# checker wants `also`, and the pre-commit hook rewrites it.
|
||||
"als",
|
||||
"ALS",
|
||||
|
||||
# ROCm's `HSA_OVERRIDE_GFX_VERSION` (Heterogeneous System Architecture); the
|
||||
# dictionary wants "HAS".
|
||||
"HSA",
|
||||
]
|
||||
@@ -99,9 +99,9 @@ introduces one, and re-evaluate entries when their `Revisit` trigger comes up.
|
||||
password prompt; `just darwin-rollback` switches the macOS generation on the machine you are on,
|
||||
and a bare `sudo systemctl restart <unit>` activates a running service the same way. The user runs
|
||||
these, not an agent.
|
||||
- `just shoryu`, `just shushou`, `just youko`, `just ruby`, `just kana`, `just lab`,
|
||||
`just k3s-test`, and `just col <tag>` activate systems through Colmena. Use the narrower recipe
|
||||
that matches the intended host scope.
|
||||
- `just shoryu`, `just shushou`, `just youko`, `just aquamarine`, `just ruby`, `just kana`,
|
||||
`just lab`, `just k3s-test`, and `just col <tag>` activate systems through Colmena. Use the
|
||||
narrower recipe that matches the intended host scope.
|
||||
- `just microvm-deploy <guest> <physical-host>` installs one MicroVM guest's runner on the physical
|
||||
host and restarts its unit there. A later host activation re-points the guest at the runner baked
|
||||
into the host's system; see WA-026 in [WORKAROUNDS.md](./WORKAROUNDS.md) before deploying. Deploy
|
||||
|
||||
+7
-3
@@ -34,8 +34,8 @@ A version or revision pinned for a reason that can go away.
|
||||
A setting that works around a bug or a hardware quirk.
|
||||
|
||||
| ID | What & where | Why | Removal condition | Added | Revisit | Status |
|
||||
| ------ | --------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------- | --------------------------------------------------------------------------------------------------- | ------ |
|
||||
| WA-008 | Relaxed NVMe APST (`nvme_core.default_ps_max_latency_us=100000`) — `hosts/idols-ai/hardware-configuration.nix` | the KINGBANK KP260 (DRAM-less QLC) froze on slow APST wake-ups | Drop the override once kernel/firmware fixes or replacement hardware make default APST reliable | 2026-10-03 | watch NVMe timeout/reset/AER in the journal | active |
|
||||
| ------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------- | --------------------------------------------------------------------------------------------------- | ------ |
|
||||
| WA-008 | Relaxed NVMe APST (`nvme_core.default_ps_max_latency_us=100000`) — `hosts/idols-ai/hardware-configuration.nix`, `hosts/idols-aquamarine/hardware-configuration.nix` | the KINGBANK KP260 (DRAM-less QLC) froze on slow APST wake-ups | Drop the override once kernel/firmware fixes or replacement hardware make default APST reliable | 2026-10-03 | watch NVMe timeout/reset/AER in the journal | active |
|
||||
| WA-009 | USB-SATA bridge mitigations (`usbcore.autosuspend=-1`, `usb-storage.delay_use=10`) — `hosts/12kingdoms-youko/default.nix` | flaky JMicron JMS567 resets with UDMA-CRC and I/O errors | Drop once the bridge/controller is replaced; the signal is smartctl CRC/media errors | 2026-09-21 | on hardware change or CRC errors | active |
|
||||
| WA-010 | Insta360 Link USB reset on S3 resume — `hosts/idols-ai/default.nix` | the camera stays enumerated with a stalled UVC endpoint after resume | Drop after the Insta360 firmware or the kernel `uvcvideo` fixes it | 2026-08-01 | on kernel or firmware change | active |
|
||||
| WA-011 | Kernel module blacklist (`esp4`, `esp6`, `rxrpc`) — `hardening/kernel-hardening.nix` | attack surface for the Dirty Frag LPE (CVE-2026-43284 / CVE-2026-43500) | Kept as defence in depth; drop only if something needs those modules | 2026-10-03 | rarely | active |
|
||||
@@ -47,6 +47,8 @@ A setting that works around a bug or a hardware quirk.
|
||||
| WA-028 | GitHub SSH pinned to IPv4 (`AddressFamily = "inet"`) — `home/base/tui/ssh.nix` | a v6 fake AAAA for `ssh.github.com` from the `2001:2::/64` pool (WA-029) is not proven on the 443 SSH tunnel yet | Drop once the v6 fake-ip pool is proven for GitHub SSH | 2026-10-10 | on a mihomo bump or a v6 pool change | active |
|
||||
| WA-029 | mihomo `fake-ip-range6` is `2001:2::/64`, not the shipped ULA default — `modules/nixos/desktop/networking/mihomo/generate.nu` | the default `fdfe:dcba:9876::/64` sits in `fc00::/7`, which Local Network Access calls local, so a public page loaded over the v4 fake pool and pulling a dual-stack subresource prompts for local-network access (Chrome 154, Firefox). The same range also stalled ssh: a session to a fake AAAA authenticated and then never closed, on both `gvisor` and `mips`. `2001:2::/48` is RFC 5180 benchmarking -- the v6 twin of the `198.18.0.0/15` v4 default, absent from that table -- and both symptoms went with the move | Go back to the upstream default once mihomo ships a pool outside the LNA non-public table; the ULA default brings the ssh stall back | 2026-10-09 | on a browser or mihomo bump -- that table already grew to cover `2001:db8::/32` and `198.18.0.0/15` | active |
|
||||
| WA-030 | mihomo unit sets `SKIP_SYSTEM_IPV6_CHECK=1` — `modules/nixos/desktop/networking/mihomo/default.nix` | mihomo checks at startup whether another NIC already holds an IPv6 address and, finding none, skips the TUN's v6 half -- but `strict-route` still installs its v6 blackhole (`ip -6 rule add priority 9000 unreachable`). On a cold boot that rule lands before networkd's Router Solicitation is answered, so the host can never send RS/NS, never gets a GUA, and mihomo never re-checks: self-locking until the core restarts. The env var forces the TUN v6 address up, which installs mihomo's normal v6 rules whose `lookup main suppress_prefixlength 0` keeps on-link neighbor discovery and the RA default route off the TUN | Drop once mihomo either re-runs the IPv6 check or skips the v6 strict-route blackhole when the v6 TUN is disabled | 2026-10-10 | on a mihomo bump | active |
|
||||
| WA-031 | `iommu=pt` kernel parameter — `hosts/idols-aquamarine/hardware-amd-gpu.nix` | ROCm's system requirements warn that multi-GPU systems may need IOMMU passthrough to prevent application hangs, and this host has two Radeon AI PRO R9700 cards. Added before the first inference run; not yet observed on gfx1201. | Drop once a ROCm release states that the default IOMMU mode is safe for multi-GPU gfx1201, or when the host runs a single GPU | 2026-10-13 | on a ROCm bump, or on the first hang report | active |
|
||||
| WA-032 | `amdgpu.runpm=0` kernel parameter — `hosts/idols-aquamarine/hardware-amd-gpu.nix` | this board's DSDT advertises ATPX power control (`\_SB.PC00.GFX0.ATPX`) but never defines the `PEG0.PEGP._OFF`/`_ON` symbols its body calls, so the first idle resume of the headless R9700 leaves it in `runtime_status = error` (`resume of IP block <smu> failed -62`) until reboot, and amdgpu discards the ACPI error. Root cause, kernel-source trail and the measured field comparison: `hosts/idols-aquamarine/ROCM.md`. | Drop once a BIOS update defines those ACPI methods (upstream releases 1023/1066 exist for this board; a flash changes PCR7, so re-enroll TPM2), or a kernel release stops trusting ATPX when its power methods are missing | 2026-10-14 | on a BIOS update or kernel bump | active |
|
||||
|
||||
If WA-008 freezes recur, the fallback is to restore the parameters commented out in
|
||||
`hosts/idols-ai/hardware-configuration.nix` —
|
||||
@@ -63,10 +65,12 @@ only delete inactive duplicates.
|
||||
A known limitation with no fix yet.
|
||||
|
||||
| ID | What & where | Why | Removal condition | Added | Revisit | Status |
|
||||
| ------ | ------------------------------------------------------------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ----------------------------------------------------------------------------------------------------------------------------- | ---------- | -------------------------------------------------------------- | ------ |
|
||||
| ------ | ---------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ----------------------------------------------------------------------------------------------------------------------------- | ---------- | -------------------------------------------------------------- | ------ |
|
||||
| WA-016 | subscription rule snapshots — `modules/nixos/desktop/networking/mihomo/` (`rules_from:`) | mihomo's `proxy-providers` imports proxies only, so the `rules` and `proxy-groups` a subscription ships with are dropped by the core; the generator re-imports them from a local YAML copy that has to be refreshed by hand when the provider changes its routing policy | Drop the snapshots once mihomo can import rules from a proxy-provider, or the providers publish their rules as rule-providers | 2026-10-04 | when mihomo gains rule import | active |
|
||||
| WA-022 | qemu-user binfmt removed — `hosts/12kingdoms-shoukei/hardware-configuration.nix` | the x86-on-arm module owns the x86 ELF handlers and this host does not need riscv64, but dropping `boot.binfmt.emulatedSystems` also drops `nix.settings.extra-platforms` and qemu-user, so `podman run --platform=linux/amd64`, x86 chroots and local builds of uncached x86 derivations need a remote x86_64 builder | Restore `boot.binfmt.emulatedSystems` (without x86_64/i686, which the module asserts against) if such a workload is needed | 2026-10-07 | when an amd64 container/chroot or uncached x86 build is needed | active |
|
||||
| WA-025 | brcmfmac 4-way-handshake offload authentication — upstream kernel/wpa_supplicant, no code in this repo | Since hostap commit 416386060, wpa_supplicant 2.11+ expects `NL80211_CMD_PORT_AUTHORIZED` from drivers that offload the 4-way handshake; brcmfmac never reports it, so authentication intermittently fails on Apple Silicon (BRCM4378/4387/4388). nixpkgs already carries the Broadcom revert in wpa_supplicant 2.12; the proper fix, Janne Grunau's `wifi: brcmfmac: cfg80211: Report port_authorized for 4-way HS offload` (2026-09), is still under review. Entering the key again after such a failure only creates a duplicate profile, so cancel and retry instead | Fix lands via a nixpkgs kernel update (or by carrying the patch through `nixpkgs-patched`) | 2026-10-07 | on every kernel bump | active |
|
||||
| WA-033 | USB-C/XHCI port `_PLD` ACPI noise — this board's DSDT, no code in this repo | every boot logs 16 `ACPI Error: Aborting method \_SB.PC00.TXHC.RHUB.SS01._PLD ... (AE_NOT_FOUND)` lines, plus `\_SB.PC02.XHCI.RHUB.HS01._PLD`; the ports work, and these messages are unrelated to the GPU's ATPX failure, which names `\_SB.PC00.GFX0.ATPX` instead | Drop if a BIOS update defines the `_PLD` descriptors | 2026-10-14 | on a BIOS update | active |
|
||||
| WA-034 | IPv6-first resolution is dead on the wired link — `hosts/idols-aquamarine/default.nix` (`networking.getaddrinfo.precedence`) | the proxy gateway answers DNS in fake-IP mode and its IPv6 path is dead: an AAAA lookup returns a fake address whose connection fails at once (measured: `curl -6` fails in 8 ms, `curl -4` answers in 0.7 s). glibc clients are covered by the RFC 6724 table with the IPv4-mapped row raised; nix's libcurl and other non-glibc resolvers ignore `/etc/gai.conf`. Disabling IPv6 with `boot.kernel.sysctl` did not work here — networkd re-enables it on the wired link for `DHCP = "ipv6"`/`IPv6AcceptRA` — so that setting was dropped on 2026-10-14 | Drop once the gateway serves proxied names over IPv6 as well (the adjacent fix is `dns.ipv6: false` in mihomo, see WA-030) | 2026-10-14 | on a mihomo or gateway change | active |
|
||||
|
||||
## WIP
|
||||
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
# Youko - Homelab Services
|
||||
|
||||
Storage, operation and maintenance related services are running on this host (migrated here from the
|
||||
retired `idols-aquamarine` guest):
|
||||
retired `aquamarine` VM — the name now belongs to the inference server, not this host):
|
||||
|
||||
1. Storage such as a git server, file server, torrent downloader, etc.
|
||||
1. Back up my personal data with restic; the data is encrypted on the client.
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{ mylib, ... }:
|
||||
{
|
||||
# The homelab service modules, migrated here from the retired
|
||||
# `hosts/idols-aquamarine` guest, now running directly on this host.
|
||||
# The homelab service modules, migrated here from the retired `aquamarine` VM
|
||||
# (the `hosts/idols-aquamarine` path is a physical host now).
|
||||
imports = mylib.scanPaths ./.;
|
||||
}
|
||||
@@ -6,10 +6,7 @@
|
||||
let
|
||||
# Powered-off hosts (SBCs) plus shoukei (exporter disabled on the
|
||||
# machine); remove entries when they come back online.
|
||||
# `aquamarine` is decommissioned: its IP now lives on youko's bridge, so
|
||||
# scraping it would relabel youko's metrics as aquamarine.
|
||||
offlineHosts = [
|
||||
"aquamarine"
|
||||
"shoukei"
|
||||
"suzu"
|
||||
"suzi"
|
||||
@@ -25,6 +22,7 @@ let
|
||||
# `vars/networking.nix`.
|
||||
physicalHosts = [
|
||||
"ai"
|
||||
"aquamarine"
|
||||
"shoryu"
|
||||
"shushou"
|
||||
"youko"
|
||||
|
||||
+24
-18
@@ -11,33 +11,39 @@ This directory contains all host-specific configurations for my NixOS and macOS
|
||||
Named after characters from "Oshi no Ko":
|
||||
|
||||
| Host | Platform | Hardware | Purpose | Status |
|
||||
| ------- | --------------- | --------------------------------------- | ------------------ | --------- |
|
||||
| `ai` | NixOS | Ultra 7 270K Plus + RTX 4090, 128G DDR5 | Gaming & Daily Use | ✅ Active |
|
||||
| ------------ | --------------- | ---------------------------------------------------- | ------------------ | --------- |
|
||||
| `ai` | NixOS | i5-13600KF + RTX 4090, 32G DDR5 | Gaming & Daily Use | ✅ Active |
|
||||
| `aquamarine` | NixOS | Ultra 7 270K Plus + 2x Radeon AI PRO R9700, 96G DDR5 | AI Inference | ✅ Active |
|
||||
| `kana` | NixOS (libvirt) | Virtual | Run AI Agents | ✅ Active |
|
||||
| `ruby` | NixOS (libvirt) | Virtual | Run AI Agents | ✅ Active |
|
||||
| `akane` | NixOS (aarch64) | Virtual (UTM) | aarch64 test VM | ✅ Active |
|
||||
|
||||
On 2026-04-27 `ai` was rebuilt on a new platform. The MSI board and i5-13600KF it was added with in
|
||||
2023-05 (the host was originally named `msi-rtx4090`) gave way to the Colorful CVN Z890 ARK FROZEN +
|
||||
Intel Core Ultra 7 270K Plus. The traces in the history are PR #257 (the NIC moved from `enp5s0` to
|
||||
`enp130s0`) and PR #258 (Niri output names renamed, `hardware-intel.nix` added). The kernel moved to
|
||||
`linuxPackages_latest` in `77ca37ea`, the same day but outside #258. The 2×48G DDR5 kit was already
|
||||
in place at that point; the repo has no record of when it went in.
|
||||
On 2026-04-27 `ai` was rebuilt on a new platform. The MSI board it was added with in 2023-05, with
|
||||
the i5-13600KF (the host was originally named `msi-rtx4090`), gave way to the Colorful CVN Z890 ARK
|
||||
FROZEN + Intel Core Ultra 7 270K Plus. The traces in the history are PR #257 (the NIC moved from
|
||||
`enp5s0` to `enp130s0`) and PR #258 (Niri output names renamed, `hardware-intel.nix` added). The
|
||||
kernel moved to `linuxPackages_latest` in `77ca37ea`, the same day but outside #258. The 2×48G DDR5
|
||||
kit was already in place at that point; the repo has no record of when it went in.
|
||||
|
||||
On 2026-10-06 `ai` grew from 2×48G (96G) to 2×48G + 2×16G (128G) across all four DIMM slots, at
|
||||
DDR5-4800 with XMP off — the only stable setting with four DIMMs on this board. The 16G pair must be
|
||||
installed first; with the 48G pair in first the board does not POST. The bring-up order, the BIOS
|
||||
settings, the stress-test recipe, and how to recover a board that will not POST are in
|
||||
[`idols-ai/MIXED-MEMORY.md`](./idols-ai/MIXED-MEMORY.md).
|
||||
installed first; with the 48G pair in first the board does not POST. It did run, in stretches of one
|
||||
to two days at a time, before a sudden hard freeze, and resuming from suspend was unreliable too;
|
||||
the stress test passed and no BIOS setting, DIMM or IMC fault was ever isolated, so 128G in four
|
||||
DIMMs stayed untrusted.
|
||||
|
||||
On 2026-10-13 that platform was split in two. The Z890 board, the 270K Plus, the 2×48G kit and two
|
||||
Radeon AI PRO R9700 cards became `aquamarine`, a headless inference server; `ai` went back to the
|
||||
MSI MAG B760M MORTAR board with the i5-13600KF, the 2×16G kit and the RTX 4090. `aquamarine` boots
|
||||
from the NVMe that used to be `ai`'s `/data` disk. Its name and the address `192.168.5.101` are
|
||||
reused from the `aquamarine` VM retired on 2026-09-21; that VM's services still run directly on
|
||||
`youko` (`hosts/12kingdoms-youko/homelab-services/`). Hardware, install and runbook:
|
||||
[`idols-aquamarine/README.md`](./idols-aquamarine/README.md).
|
||||
|
||||
##### Retired
|
||||
|
||||
| Host | Platform | Hardware | Purpose | Status |
|
||||
| ------------ | --------------- | -------- | --------------------- | ---------- |
|
||||
| `aquamarine` | NixOS (libvirt) | Virtual | Monitoring & Services | ⚪ Retired |
|
||||
|
||||
`aquamarine` is retired; its services now run directly on `youko`
|
||||
(`hosts/12kingdoms-youko/homelab-services/`).
|
||||
_No physical host in this group is retired. The `aquamarine` VM (NixOS on libvirt, monitoring and
|
||||
services) was retired on 2026-09-21 and its name was reused for the inference server above._
|
||||
|
||||
#### `darwin` - macOS Systems
|
||||
|
||||
@@ -124,7 +130,7 @@ just build-host <host>
|
||||
|
||||
`eval-host` only evaluates the configuration. `build-host` builds the system closure without
|
||||
activating it. Host activation is always explicit: Colmena (`just shoryu`, `just shushou`,
|
||||
`just youko`) from another machine, or `just local` on the machine itself.
|
||||
`just youko`, `just aquamarine`) from another machine, or `just local` on the machine itself.
|
||||
|
||||
## Distributed Building
|
||||
|
||||
|
||||
@@ -0,0 +1,235 @@
|
||||
# Host - Aquamarine
|
||||
|
||||
Headless AI inference server. Revives the `aquamarine` name, retired on 2026-09-21 when the VM of
|
||||
that name was decommissioned; this is a different, physical machine that reuses the name and the
|
||||
`192.168.5.101` address.
|
||||
|
||||
## Hardware
|
||||
|
||||
| Part | Model | Notes |
|
||||
| ------ | ----------------------------------------------------- | ------------------------------------------------------- |
|
||||
| Board | Colorful CVN Z890 ARK FROZEN V20 (BIOS 1013) | moved from `ai` on 2026-10-13, so its EFI vars come too |
|
||||
| CPU | Intel Core Ultra 7 270K Plus | Arrow Lake; NPU unused here |
|
||||
| Memory | 2x48G JUHOR (96G), 4800 MT/s | the 2x16G kit stayed with `ai` |
|
||||
| GPU | 2x AMD Radeon AI PRO R9700, 32G each (RDNA4, gfx1201) | 64G of VRAM total |
|
||||
| Root | Fanxiang S790 2TB | was `ai`'s `/data` disk until 2026-10-13 |
|
||||
| NIC | Realtek RTL8125BG 2.5GbE, `enp133s0` today | matched by the `en*` prefix |
|
||||
|
||||
Two things about this board set the ceiling for the GPU work:
|
||||
|
||||
- The second full-length x16 slot is chipset-attached: its root port negotiates Gen4 x4, while the
|
||||
card side reports Gen5 x16 either way (./ROCM.md has the link measurements). The chipset card
|
||||
probes **7.1 GB/s** host->device against 46.3 GB/s for the CPU-attached card, so a two-card layer
|
||||
split is 8-27% slower per request and ~9% ahead on six concurrent requests. Read the inference
|
||||
engine's `PCIe probe` line rather than `lspci`, and prefer a CPU-attached slot for a second card.
|
||||
- Multi-GPU work should split by layer (RPC / pipeline parallel), not by tensor parallelism: the
|
||||
R9700 has no xGMI link between cards, and a dual-R9700 report has RCCL deadlocking with vLLM TP=2
|
||||
even with `NCCL_P2P_DISABLE=1`. See [ROCM.md](./ROCM.md).
|
||||
|
||||
The iGPU needs no configuration: the kernel binds `i915` and
|
||||
`hardware.enableRedistributableFirmware` is on, so a monitor plugged into the board's HDMI/DP ports
|
||||
gets the text console (`multi-user.target`, no display manager) when SSH is unreachable.
|
||||
|
||||
`iommu=pt` is set in [hardware-amd-gpu.nix](./hardware-amd-gpu.nix) because ROCm documents hangs on
|
||||
multi-GPU systems without IOMMU passthrough. The same file disables amdgpu runtime power management:
|
||||
this board's DSDT advertises ATPX power control that its own ACPI methods cannot deliver, so the
|
||||
first idle resume of the headless R9700 kills it until reboot. [ROCM.md](./ROCM.md) records whose
|
||||
defect that is, the supported ROCm versions, and the inference notes.
|
||||
|
||||
## Disk layout
|
||||
|
||||
Declarative via [disko-fs.nix](./disko-fs.nix): ESP + LUKS (`aqua-luks`) + btrfs subvolumes (`@nix`,
|
||||
`@persistent`, `@snapshots`, `@tmp`, `@swap` with a 96G swapfile), ephemeral tmpfs root, state under
|
||||
`/persistent` via [preservation.nix](./preservation.nix). Secure Boot via lanzaboote
|
||||
([secureboot.nix](./secureboot.nix)).
|
||||
|
||||
The mapper is `aqua-luks`, not `nixos-luks`: the disk is formatted while the staging host is
|
||||
running, and that host already owns the `nixos-luks` name.
|
||||
|
||||
## Install (staged from a running host)
|
||||
|
||||
This disk is installed into a machine that is not the one it gets formatted on: `disko` runs while
|
||||
the disk is still attached to a running NixOS host (`ai` below), and the disk then moves to its own
|
||||
machine. The target machine is the board the install runs on, so the EFI boot entry and the Secure
|
||||
Boot key enrollment the host already has carry over. That is what makes a staged install possible;
|
||||
it also means these steps run on a machine whose own root disk is one `disko destroy` away.
|
||||
|
||||
Steps 1 to 6 only touch this disk and leave `ai` running. `ai`'s own deploy and shutdown are step 7,
|
||||
and they belong together: the machine comes back on the new board with this disk gone.
|
||||
|
||||
1. Confirm the target disk and release it: `lsblk -f` must show no mount and no open mapper on it. A
|
||||
disk reused from another role still carries that role's mounts:
|
||||
|
||||
```bash
|
||||
ls -l /dev/disk/by-id/nvme-Fanxiang_S790_2TB_FXS790254050582
|
||||
lsblk -f
|
||||
sudo umount /data # the previous role's mountpoint
|
||||
sudo cryptsetup close data-luks # and its mapper
|
||||
```
|
||||
|
||||
2. Partition, encrypt and mount. This **destroys everything on that disk** — re-read the `device`
|
||||
line in [disko-fs.nix](./disko-fs.nix) first. Run it as **root**: unprivileged, disko still walks
|
||||
through the confirmation prompt and then fails on every step with `Permission denied`.
|
||||
|
||||
```bash
|
||||
cd ~/nix-config
|
||||
sudo nix run github:nix-community/disko -- --mode destroy,format,mount hosts/idols-aquamarine/disko-fs.nix
|
||||
```
|
||||
|
||||
Success looks like two new partitions — `disk-nixos-aquamarine-ESP` (vfat, 600M) and
|
||||
`disk-nixos-aquamarine-root` (crypto_LUKS) — and seven mounts under `/mnt`. Check before moving
|
||||
on:
|
||||
|
||||
```bash
|
||||
lsblk -o NAME,SIZE,FSTYPE,PARTLABEL /dev/nvme0n1
|
||||
findmnt -R /mnt
|
||||
```
|
||||
|
||||
3. Reuse the staging host's Secure Boot PKI bundle, so lanzaboote signs this system with a key the
|
||||
firmware already trusts. The bundle contains a private signing key: copy it yourself, and never
|
||||
commit it or send it through chat.
|
||||
|
||||
```bash
|
||||
sudo mkdir -p /mnt/persistent/etc/secureboot
|
||||
sudo cp -a /persistent/etc/secureboot/. /mnt/persistent/etc/secureboot/
|
||||
|
||||
# lanzaboote reads the key from /etc/secureboot inside the target, and preservation
|
||||
# only bind-mounts that directory at boot — so expose it during the install too.
|
||||
sudo mkdir -p /mnt/etc/secureboot
|
||||
sudo mount --bind /mnt/persistent/etc/secureboot /mnt/etc/secureboot
|
||||
```
|
||||
|
||||
`nixos-install` installs the boot loader through `nixos-enter --root /mnt` and
|
||||
`NIXOS_INSTALL_BOOTLOADER=1 ... switch-to-configuration boot`, so lanzaboote's install hook reads
|
||||
`/etc/secureboot/keys/db/db.key` **from the target root**. Without the bind mount that path is an
|
||||
empty directory and the install fails at "installing the boot loader", however good the copy in
|
||||
`/mnt/persistent` is.
|
||||
|
||||
Skipping this is fine only if you would rather start over: delete the firmware keys,
|
||||
`sbctl create-keys` + `sbctl enroll-keys` in setup mode. The fresh keys must end up in
|
||||
`/etc/secureboot` — lanzaboote's `pkiBundle`, while `sbctl create-keys` writes to its own keydir
|
||||
— or the firmware trusts a key this system's UKI was not signed with. That changes PCR7, so any
|
||||
TPM2 LUKS enrollment made before it stops unlocking.
|
||||
|
||||
4. Enroll the new LUKS volume with this board's TPM2, keeping the passphrase slot as the fallback:
|
||||
|
||||
```bash
|
||||
sudo systemd-cryptenroll --tpm2-device=auto --tpm2-pcrs=0+7 \
|
||||
/dev/disk/by-id/nvme-Fanxiang_S790_2TB_FXS790254050582-part2
|
||||
sudo systemd-cryptenroll /dev/disk/by-id/nvme-Fanxiang_S790_2TB_FXS790254050582-part2 # slots: expect tpm2 + passphrase
|
||||
```
|
||||
|
||||
PCR7 measures the Secure Boot key database, so enroll **after** step 3, not before. Re-run this
|
||||
whenever the Secure Boot keys change.
|
||||
|
||||
5. Install the real system. The closure is mostly cached on the staging host, and step 3's bind
|
||||
mount is what lets lanzaboote sign into this disk's ESP:
|
||||
|
||||
```bash
|
||||
cd ~/nix-config
|
||||
sudo nixos-install --root /mnt --flake .#aquamarine --no-root-password
|
||||
```
|
||||
|
||||
If a private flake input is not in the host's store yet, fetch it as root: `sudo` does not pass
|
||||
`SSH_AUTH_SOCK` through on this host, so run the install from a root shell with its own agent
|
||||
(`sudo su`, then `eval "$(ssh-agent)" && ssh-add ~/.ssh/<github-key>`).
|
||||
|
||||
The installer writes an EFI boot entry into **this board's NVRAM** and may put it first. Do not
|
||||
reboot the staging host with both disks attached — it would boot aquamarine's system.
|
||||
|
||||
The install generates the target's `/etc`, but preservation only bind-mounts `/persistent` at
|
||||
boot, so the state it looks for has to be in place before the disk is unmounted. Move the
|
||||
generated `/etc/ssh` and `machine-id` in, then read the listing:
|
||||
|
||||
```bash
|
||||
sudo nixos-enter --root /mnt
|
||||
mkdir -p /persistent/etc
|
||||
mv /etc/ssh /persistent/etc/
|
||||
mv /etc/machine-id /persistent/etc/
|
||||
ls -l /persistent/etc/ssh /persistent/etc/ssh/authorized_keys.d
|
||||
exit
|
||||
```
|
||||
|
||||
`authorized_keys.d/<user>` must be a **regular file** with mode `0444`: sshd's `StrictModes`
|
||||
rejects a key file whose path crosses `/nix/store`, which is group-writable (`1775`), so a
|
||||
symlink to `/etc/static/...` there locks every login out. `sshd_config` is a symlink to
|
||||
`/etc/static/ssh/sshd_config` on purpose — that one tracks the current generation. If a file is
|
||||
missing, copy the real one in and create its directory if needed:
|
||||
`cp -L /etc/static/ssh/authorized_keys.d/. /persistent/etc/ssh/authorized_keys.d/`.
|
||||
|
||||
6. Unmount the target: `umount -R /mnt` also drops step 3's bind mount. This disk is done; leave the
|
||||
staging host running for step 7.
|
||||
|
||||
```bash
|
||||
sudo umount -R /mnt
|
||||
findmnt -R /mnt # nothing left mounted
|
||||
```
|
||||
|
||||
7. On `ai`, stage its next generation **and** shut it down — one step, never one without the other.
|
||||
The machine boots on the new board with this disk gone, so the generation it comes back on must
|
||||
not wait for a LUKS volume here; and staging only takes effect at that boot, so shutting down
|
||||
first would leave the current generation as the default.
|
||||
|
||||
```bash
|
||||
# the host's initrd must no longer list this disk's mapper
|
||||
nix eval --json .#nixosConfigurations.ai-niri.config.boot.initrd.luks.devices --apply builtins.attrNames
|
||||
# [ "nixos-luks" ]
|
||||
|
||||
just niri boot # stage only: `switch` would apply the NIC rename now and cut the LAN
|
||||
sudo poweroff # only after the deploy succeeded
|
||||
```
|
||||
|
||||
Then move the disk to its machine, in the same M.2 slot: the EFI entry names the disk by its
|
||||
device path, and a different slot can invalidate it.
|
||||
|
||||
## First boot on aquamarine
|
||||
|
||||
1. The initrd should unlock `aqua-luks` from the TPM; if it prompts, use the passphrase.
|
||||
2. If the firmware has no boot entry for the disk, pick it once from the boot menu (the ESP also
|
||||
carries the `\EFI\BOOT\BOOTX64.EFI` fallback), then let the first activation write the entry.
|
||||
3. From another host:
|
||||
|
||||
```bash
|
||||
ssh aquamarine ip addr
|
||||
ssh aquamarine systemctl --failed
|
||||
ssh aquamarine journalctl -b -p err
|
||||
ssh aquamarine rocminfo
|
||||
ssh aquamarine rocm-smi --showproductname
|
||||
```
|
||||
|
||||
If `ssh` refuses the connection, or `sshd` logs `bad ownership or mode for directory /nix/store`,
|
||||
the `/etc/ssh` copy under `/persistent` is incomplete — repeat step 5's check from the console
|
||||
before deploying anything else.
|
||||
|
||||
`ip addr` must show `192.168.5.101` on the wired interface. The networkd match uses the `en*`
|
||||
prefix, so a renamed NIC cannot strand the host; if the address is missing, check the cable and
|
||||
`ip -br link` first.
|
||||
|
||||
## After the first boot
|
||||
|
||||
The install created a fresh SSH host key; the old `aquamarine` VM key is dead and the new one is
|
||||
already in `vars/networking.nix` under `knownHosts` (2026-10-13). Redo these steps only after a
|
||||
reinstall or a key rotation:
|
||||
|
||||
1. `ssh aquamarine cat /etc/ssh/ssh_host_ed25519_key.pub`.
|
||||
2. Put that key in `vars/networking.nix` under `knownHosts` (the entry was removed with the old key)
|
||||
and, if this host should read agenix secrets, in `~/codes/nix-secrets/secrets.nix` under
|
||||
`application_server_keys`.
|
||||
3. Rekey and bump the pin: `just -f secrets/Justfile rekey`, push `nix-secrets`, then
|
||||
`just upp mysecrets` here.
|
||||
|
||||
This config enables no `modules.secrets.*` group, so aquamarine decrypts nothing today; adding a
|
||||
group is what requires the recipient step above.
|
||||
|
||||
## Deploy
|
||||
|
||||
```bash
|
||||
just aquamarine # colmena, from any other host
|
||||
just local # on aquamarine itself
|
||||
```
|
||||
|
||||
## Rollback
|
||||
|
||||
The install only touches the target disk; the staging host keeps booting from its own. To undo, move
|
||||
the disk back, restore the `hosts/idols-ai/disko-fs-data.nix` module this change deleted, and
|
||||
re-enroll that machine's TPM2 slot — see `../idols-ai/README.md`.
|
||||
@@ -0,0 +1,273 @@
|
||||
# ROCm on aquamarine
|
||||
|
||||
`aquamarine` runs Radeon AI PRO R9700 cards (Navi 48, RDNA4, PCI ID `1002:7551`, ROCm target
|
||||
`gfx1201`) as headless compute devices; the Intel iGPU drives the console. This note records which
|
||||
versions support the card, why amdgpu runtime power management leaves it unusable, and which
|
||||
decisions that produced. Claims are marked **verified** (primary source), **community-reported**, or
|
||||
**inference** (read from kernel source, not confirmed by a maintainer).
|
||||
|
||||
## Versions
|
||||
|
||||
- **ROCm**: AMD's compatibility matrix has no RDNA4 content and no `gfx1201` in the 6.4.0 column;
|
||||
`gfx1201` and the RDNA4 row appear from 7.0.0 on. The R9700 first appears in AMD's _Radeon_ matrix
|
||||
for **6.4.4** (Ubuntu 24.04.2/22.04.5, RHEL 9.6/9.4), so it arrived inside 6.4.x, not at 6.4.0.
|
||||
**verified**
|
||||
([matrix](https://github.com/ROCm/legacy-rocm-build/blob/develop/docs/compatibility/compatibility-matrix.rst),
|
||||
[6.4.4 Radeon matrix](https://rocm.docs.amd.com/projects/radeon-ryzen/en/docs-6.4.4/docs/compatibility/compatibilityrad/native_linux/native_linux_compatibility.html))
|
||||
- **ROCm today**: system requirements list `AMD Radeon AI PRO R9700 | RDNA4 | gfx1201 | ✅`.
|
||||
**verified**
|
||||
([requirements](https://rocm.docs.amd.com/projects/install-on-linux/en/latest/reference/system-requirements.html))
|
||||
- **This repository**: nixpkgs `f45c6f0` provides `rocmPackages.clr` and `rocmPackages.rocm-smi`
|
||||
7.2.3 and Mesa 26.2.4; the host runs the nixpkgs default kernel (6.18.x LTS), not
|
||||
`linuxPackages_latest` (7.2.8), which is what the first bring-up booted. **verified** (checked by
|
||||
`nix eval`)
|
||||
- **Mesa**: "initial GFX12 (RDNA4) support on RADV" landed in 25.0.0. **verified**
|
||||
([notes](https://gitlab.freedesktop.org/mesa/mesa/-/blob/main/docs/relnotes/25.0.0.rst))
|
||||
- **Kernel**: amdgpu has no static `1002:7551` entry; its ID table ends with
|
||||
`PCI_DEVICE(0x1002, PCI_ANY_ID)` wildcards carrying `CHIP_IP_DISCOVERY`, so binding depends on the
|
||||
`soc24`/`gfx_v12_0` IP blocks and firmware, not a PCI ID. AMD validates Ubuntu 24.04.3 HWE kernel
|
||||
6.14 for R9700 with ROCm 6.4.4; no primary source states a minimum kernel for Navi 48.
|
||||
**verified**
|
||||
([amdgpu_drv.c](https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git/tree/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c?h=v7.2.8))
|
||||
- **linux-firmware**: Navi 4x uses `gc_12_0_1_*`, `psp_14_0_3_*`, `smu_14_0_3.bin`,
|
||||
`sdma_7_0_1.bin`, `dcn_4_0_1_dmcub.bin`; an Ubuntu SRU "Fix power state transition on navi4x"
|
||||
updates exactly those files and states they fix bad D3cold→D0 transitions. **verified**
|
||||
([LP #2122659](https://bugs.launchpad.net/ubuntu/+source/linux-firmware/+bug/2122659))
|
||||
|
||||
## The runtime-PM failure
|
||||
|
||||
The chain is readable from kernel source: a defect in this board's ACPI firmware that the driver
|
||||
walks into and then hides. The R9700 and the ROCm userspace are fine.
|
||||
|
||||
The defect is in Colorful's DSDT. It advertises the ATPX dGPU power-control method while the method
|
||||
body calls `\_SB.PC00.PEG0.PEGP._OFF`/`_ON`, symbols this DSDT never defines, so the firmware
|
||||
promises power control it cannot deliver. amdgpu then hides the failure instead of surfacing it: it
|
||||
routes runtime PM through ATPX and discards the ACPI error (step 4), which turns "this firmware
|
||||
cannot power-gate the card" into "the card is dead until reboot".
|
||||
|
||||
Hitting it took exactly two display-class devices (the iGPU kept for the console fallback plus one
|
||||
R9700) for `amdgpu_atpx_detect()` to accept ATPX, this DSDT, and an idle dGPU with no display
|
||||
attached. With **two** R9700s installed there are three display-class devices, so that condition no
|
||||
longer holds and `amdgpu.runpm=0` may be belt-and-braces rather than a live fix; it stays because
|
||||
disabling runtime PM on an always-on host costs nothing and the failure needs a reboot to clear. The
|
||||
field comparison below predates the second card.
|
||||
|
||||
1. `amdgpu_atpx_detect()` accepts an ACPI `ATPX` method only when two VGA/display-class devices
|
||||
exist, which the iGPU plus one R9700 satisfies; this DSDT does expose `\_SB.PC00.GFX0.ATPX`, as
|
||||
the log names it. **verified**
|
||||
([atpx handler](https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git/tree/drivers/gpu/drm/amd/amdgpu/amdgpu_atpx_handler.c?h=v7.2.8))
|
||||
2. When ATPX advertises dGPU power control, amdgpu sets `AMD_IS_PX` on the R9700 and, with the
|
||||
default `runpm=-1`, selects `AMDGPU_RUNPM_PX` ("Using ATPX for runtime pm"). **verified**
|
||||
([amdgpu_device.c](https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git/tree/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c?h=v7.2.8))
|
||||
3. `vga_switcheroo_init_domain_pm_ops()` routes runtime suspend/resume through the ATPX handler,
|
||||
which evaluates `\_SB.PC00.PEG0.PEGP._OFF`/`_ON`. Those symbols are absent here, so ACPI aborts
|
||||
the method with `AE_NOT_FOUND`. **verified**
|
||||
([vga_switcheroo.c](https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git/tree/drivers/gpu/vga/vga_switcheroo.c?h=v7.2.8))
|
||||
4. The failure is discarded twice over: `amdgpu_atpx_power_state()` calls
|
||||
`amdgpu_atpx_set_discrete_state()` and then returns 0 unconditionally, and
|
||||
`vga_switcheroo_power_switch()` never looks at the handler's return value either. So the failed
|
||||
switch is silent; the driver resumes as if the card had been power-cycled, re-initializes SMU,
|
||||
gets no answer (`SMU: No response msg_reg: 6 resp_reg: 0`) and reports
|
||||
`resume of IP block <smu> failed -62` with `runtime_status = error`. **verified** for the two
|
||||
discarded returns, **inference** for the SMU sequence that follows
|
||||
([amdgpu_atpx_handler.c](https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git/tree/drivers/gpu/drm/amd/amdgpu/amdgpu_atpx_handler.c?h=v6.18.54),
|
||||
[vga_switcheroo.c](https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git/tree/drivers/gpu/vga/vga_switcheroo.c?h=v6.18.54)).
|
||||
5. With `runtime_status` not `active`, rocm-smi prints its low-power-state warning;
|
||||
`check_runtime_status()` reads `…/device/power/runtime_status`. **verified**
|
||||
([rocm_smi.py](https://github.com/ROCm/rocm-systems/blob/rocm-7.2.3/projects/rocm-smi-lib/python_smi_tools/rocm_smi.py))
|
||||
|
||||
**No upstream report of this exact chain was found.** Searches of the freedesktop drm/amd tracker,
|
||||
kernel Bugzilla and distro trackers found no bug matching `PEGP._OFF` + `AE_NOT_FOUND` +
|
||||
`Aborting method … ATPX` on RDNA4 desktop hardware. The nearest documents are the amd-gfx patch
|
||||
"drm/amdgpu/soc24: reset dGPU if suspend got aborted", which handles the same `SMU … -62` tail but
|
||||
only when `adev->in_s3` is set (present in 7.2.8; it does **not** cover runtime PM), and the navi4x
|
||||
firmware SRU. **verified**
|
||||
([patch](https://lists.freedesktop.org/archives/amd-gfx/2026-June/146878.html),
|
||||
[soc24.c](https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git/tree/drivers/gpu/drm/amd/amdgpu/soc24.c?h=v7.2.8))
|
||||
|
||||
Nor is a fix in flight on the code itself: `amdgpu_atpx_handler.c`'s most recent change is a 2026-05
|
||||
buffer-length hardening, and its dGPU power-control semantics have not been touched since 2019,
|
||||
while the 2025-2026 `vga_switcheroo.c` changes are cosmetic (`pci_is_display()`, fbcon wrappers,
|
||||
`kmalloc_obj` conversions). **verified** (mainline commit history of both files)
|
||||
|
||||
## What actually fixes it
|
||||
|
||||
- `amdgpu.runpm=0` is the documented "disable" value (default `-1`, auto). With mode `NONE`,
|
||||
`amdgpu_pmops_runtime_checks()` calls `pm_runtime_forbid()` and returns `-EBUSY`, the
|
||||
vga_switcheroo domain is not installed, and no runtime suspend can run. **verified**
|
||||
([amdgpu_drv.c](https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git/tree/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c?h=v7.2.8))
|
||||
- `echo on > /sys/bus/pci/devices/…/power/control` is the PM-core equivalent — the device never
|
||||
runtime-suspends — but it leaves the driver registered as a PX client, which is why this host sets
|
||||
`amdgpu.runpm=0` instead (below) rather than persisting that write with a udev rule. **verified**
|
||||
for the ABI, **community-reported** as a fix.
|
||||
- AMD's own notes carry the workaround ("users can run amdgpu.runpm=0, which temporarily disables
|
||||
the runtime power management feature from the driver"), and the Arch Wiki gives the same advice
|
||||
for a dGPU that "becomes disabled during use", citing drm/amd issue 1820. **verified**
|
||||
([ROCm 5.0.0 highlights](https://github.com/ROCm/legacy-rocm-build/blob/develop/tools/autotag/templates/highlights/5.0.0.md),
|
||||
[Arch Wiki](https://wiki.archlinux.org/title/AMDGPU#Issues_with_power_management_/_dynamic_re-activation_of_a_discrete_amdgpu_graphics_card))
|
||||
|
||||
The workaround is field-verified on this host: the pre-fix boot logged
|
||||
`amdgpu: Using ATPX for runtime pm` and left the card in `runtime_status = error`, while the 6.18.54
|
||||
boot with `amdgpu.runpm=0` logs `amdgpu: runtime pm is manually disabled` and
|
||||
`amdgpu: Runtime PM not available`, keeps `power/control = on` and `runtime_status = active`, and
|
||||
`rocm-smi` reads the card normally after minutes of idling. **verified** (`journalctl -k -b 0` vs
|
||||
`-b -2` on this host)
|
||||
|
||||
One adjacent problem gets conflated with this one: Navi 48 s2idle breaks when
|
||||
`hardware.amdgpu.overdrive`'s default `ppfeaturemask = "0xfffd7fff"` disables GFXOFF (upstream
|
||||
default `0xfff7bfff`; `0xfff7ffff` works). **verified**
|
||||
([nixpkgs#564601](https://github.com/NixOS/nixpkgs/issues/564601))
|
||||
|
||||
`amdgpu.ppfeaturemask=0xffffffff` does not address the ATPX/SMU path; it comes from a community
|
||||
tuning repository and no primary source ties it to it. **community-reported**
|
||||
([r9700-setup](https://github.com/stew675/r9700-setup/blob/HEAD/README.md)). The host's own
|
||||
`pcie_aspm=off` and `processor.max_cstate=2` are unrelated to this path too — they are set for PCIe
|
||||
link retraining and C-state exit latency, with their reasons in
|
||||
[hardware-amd-gpu.nix](./hardware-amd-gpu.nix).
|
||||
|
||||
## Kernel parameters and module options
|
||||
|
||||
- `iommu=pt` — keep. AMD: "Systems with multiple GPUs may require iommu=pt … to prevent application
|
||||
hangs". **verified**
|
||||
([requirements](https://rocm.docs.amd.com/projects/install-on-linux/en/latest/reference/system-requirements.html))
|
||||
- `iommu=off` — claimed by `stew675/r9700-setup` to "allow for P2P between multiple R9700's"; no AMD
|
||||
statement supports that. **community-reported**
|
||||
- `amdgpu.gpu_recovery=1` — enables GPU recovery; default is `-1`, "auto, disabled except SRIOV".
|
||||
**verified**
|
||||
([amdgpu_drv.c](https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git/tree/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c?h=v7.2.8))
|
||||
- `amdgpu.ppfeaturemask` — exposed as `hardware.amdgpu.overdrive.ppfeaturemask`; see the s2idle
|
||||
caveat above. **verified**
|
||||
([module](https://github.com/NixOS/nixpkgs/blob/master/nixos/modules/services/hardware/amdgpu.nix))
|
||||
- `amdgpu.ras_enable` — the real parameter name (`module_param_named(ras_enable, …)`), default `-1`;
|
||||
the driver's GECC message calls it `amdgpu_ras_enable`. **verified**
|
||||
([amdgpu_psp.c](https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git/tree/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c?h=v7.2.8))
|
||||
|
||||
The Arch Wiki documents the same knobs plus `power_dpm_force_performance_level` and the COMPUTE
|
||||
profile, but has no R9700/gfx1201 entry; its ROCm section only says the official list is short and
|
||||
equivalent-generation consumer cards are "known to work", and never mentions
|
||||
`HSA_OVERRIDE_GFX_VERSION`. **verified** ([AMDGPU](https://wiki.archlinux.org/title/AMDGPU),
|
||||
[GPGPU § ROCm](https://wiki.archlinux.org/title/General-purpose_computing_on_graphics_processing_units#ROCm))
|
||||
|
||||
## ROCm userland in nixpkgs
|
||||
|
||||
- `hardware.amdgpu.opencl.enable = true` adds both `rocmPackages.clr` and `rocmPackages.clr.icd` —
|
||||
the module's own shape — rather than `.icd` alone. **verified**
|
||||
([amdgpu.nix](https://github.com/NixOS/nixpkgs/blob/master/nixos/modules/services/hardware/amdgpu.nix))
|
||||
- **`rocm-smi` works on a healthy card.** The `Fail to open libdrm_amdgpu.so.1` message upstream
|
||||
logs (and AMD's amdsmi PR tolerates, closed unmerged) appeared here only while the R9700 sat in
|
||||
`runtime_status = error`, so it is a symptom of the wedged card rather than a packaging bug. With
|
||||
`amdgpu.runpm=0` the tool reads temperatures, clocks, power and VRAM, and `--showproductname`
|
||||
answers `Card Model: 0x7551`, `Card Vendor: 0x1002`, `GFX Version: gfx1201`; only `Card Series`
|
||||
stays `N/A` because the userspace name table has no R9700 entry. **verified** (observed on this
|
||||
host) ([amdsmi PR](https://github.com/ROCm/rocm-systems/pull/2987))
|
||||
- nixpkgs' Ollama and Open WebUI modules use `SupplementaryGroups = [ "render" ]`, matching this
|
||||
host. **verified**
|
||||
([ollama.nix](https://github.com/NixOS/nixpkgs/blob/master/nixos/modules/services/misc/ollama.nix))
|
||||
- A real NixOS RDNA4 host pins `linuxPackages_latest` and sets `amdgpu.runpm=0` ("Disable runtime
|
||||
power management for stability") plus `amdgpu.gpu_recovery=1` and `amdgpu.dc=1`.
|
||||
**community-reported**
|
||||
([config](https://github.com/dustinlyons/nixos-config/blob/main/hosts/nixos/default.nix))
|
||||
- GitHub code search found **no** public Nix configuration targeting `gfx1201` or the R9700.
|
||||
|
||||
## Two R9700s
|
||||
|
||||
- R9700 cards have no Infinity Fabric/xGMI link. A dual-R9700 report documents an **RCCL deadlock
|
||||
with vLLM TP=2** on ROCm 7.2.1 (RCCL 2.27.7): both GPUs spin at 100%, `NCCL_P2P_DISABLE=1` changes
|
||||
nothing, and the same stack worked with RCCL 2.27.3. **verified** (open upstream)
|
||||
([rocm-systems#5480](https://github.com/ROCm/rocm-systems/issues/5480))
|
||||
- llama.cpp gates peer copies behind `GGML_CUDA_NO_PEER_COPY`; without it cross-device copies use
|
||||
`cudaMemcpyPeerAsync` and the backend advertises P2P events. The tuning repo recommends
|
||||
`-DGGML_CUDA_NO_PEER_COPY=1` "if P2P is unstable in your topology". **verified** (flag),
|
||||
**community-reported** (recommendation)
|
||||
([ggml-cuda.cu](https://github.com/ggml-org/llama.cpp/blob/master/ggml/src/ggml-cuda/ggml-cuda.cu))
|
||||
- `HIP_VISIBLE_DEVICES` (documented by llama.cpp) and `ROCR_VISIBLE_DEVICES` (used as
|
||||
`ROCR_VISIBLE_DEVICES=0` in the R9700 report) both hide the iGPU. **verified**
|
||||
([build docs](https://github.com/ggml-org/llama.cpp/blob/master/docs/build.md),
|
||||
[legacy-rocm-build#6630](https://github.com/ROCm/legacy-rocm-build/issues/6630))
|
||||
- Split by layer/pipeline, not tensor parallel: `-sm/--split-mode` offers `none`, `layer` (default,
|
||||
"split layers and KV across GPUs (pipelined)"), `row` and `tensor` (parallelised, EXPERIMENTAL);
|
||||
`--device` and `--main-gpu` choose devices. **verified**
|
||||
([arg.cpp](https://github.com/ggml-org/llama.cpp/blob/master/common/arg.cpp))
|
||||
- Both installed R9700s (`0000:04:00.0` behind the card's own switch at `02:00.0`/`03:00.0`, and
|
||||
`0000:83:00.0` behind `82:00.0`) negotiate `current_link_width = 16` at `32.0 GT/s` (PCIe 5.0 x16,
|
||||
their maximum) — that is the card-side link, not the slot. The slot links differ: the CPU root
|
||||
port `0000:00:06.0` is x16 at 32 GT/s, while the chipset root port `0000:80:1c.0` is **x4 at 16
|
||||
GT/s (PCIe 4.0)**. The board manual's x4 claim is therefore measured, and because `lspci` on the
|
||||
card reports x16 either way, read the root port or the engine's `PCIe probe` line (./README.md).
|
||||
|
||||
## Inference software
|
||||
|
||||
- **llama.cpp**: `-DGGML_HIP=ON -DGPU_TARGETS=gfx1201` (`GPU_TARGETS` is optional; omitting it
|
||||
builds for all GPUs present). `HSA_OVERRIDE_GFX_VERSION` is documented only for GPUs that are not
|
||||
officially supported, so it is not needed. ROCm's matrix validates llama.cpp `b6356` with 7.0.x.
|
||||
**verified** ([build docs](https://github.com/ggml-org/llama.cpp/blob/master/docs/build.md),
|
||||
[matrix](https://github.com/ROCm/legacy-rocm-build/blob/develop/docs/compatibility/compatibility-matrix.rst))
|
||||
- **llama.cpp on gfx1201 is not clean**: an upstream report on one R9700 shows Vulkan passing while
|
||||
ROCm 7.14 hangs in generation and later resets MES/PSP (`resume of IP block <psp> failed -62`),
|
||||
still open. **verified** ([#6630](https://github.com/ROCm/legacy-rocm-build/issues/6630))
|
||||
- The tuning repo also passes `-DGGML_HIP_ROCWMMA_FATTN=ON` and `-DGGML_HIP_RCCL=ON`; only
|
||||
`GGML_HIP_RCCL` exists in `ggml/CMakeLists.txt`, so the ROCWMMA flash-attention flag is stale and
|
||||
CMake ignores it. **community-reported**
|
||||
([CMakeLists.txt](https://github.com/ggml-org/llama.cpp/blob/master/ggml/CMakeLists.txt))
|
||||
- **Ollama**: the PR "Add native support for AMD RDNA 4 (gfx1201 / Radeon AI Pro R9700)" whitelists
|
||||
the architecture and adds ROCm targets but is **closed and not merged**; it states RDNA4 discovery
|
||||
timed out in the HIP handshake before whitelisting and links reports of
|
||||
`TensileLibrary_lazy_gfx1201.dat` missing. A later report shows Ollama 0.20.2-rocm detecting
|
||||
`compute=gfx1201` while still logging the rocBLASLt `.dat` error. **verified**
|
||||
([#14021](https://github.com/ollama/ollama/pull/14021),
|
||||
[#15338](https://github.com/ollama/ollama/issues/15338))
|
||||
- **vLLM**: the current ROCm GPU installation page does not mention gfx1201, RDNA4, R9700 or 9070;
|
||||
working dual-R9700 setups are community forks. **verified** (docs' silence)
|
||||
([vLLM docs](https://docs.vllm.ai/en/latest/getting_started/installation/gpu.html));
|
||||
**community-reported** ([toolboxes](https://github.com/kyuz0/amd-r9700-vllm-toolboxes))
|
||||
|
||||
## Power, thermals, ECC
|
||||
|
||||
- **GECC** is a PSP boot-config bit: the driver prints `GECC is enabled`, or
|
||||
`GECC will be enabled in next boot cycle` after setting it; enabling needs UMC RAS support and
|
||||
`ras_default_ecc_enabled`/`amdgpu.ras_enable=1`. It takes effect next boot and costs VRAM
|
||||
capacity/bandwidth, so decide before filling the cards. **verified**
|
||||
([amdgpu_psp.c](https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git/tree/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c?h=v7.2.8))
|
||||
- **Monitoring/control**: `power_dpm_force_performance_level`, `pp_dpm_*` clocks,
|
||||
`gpu_busy_percent`, `mem_busy_percent` and the `fan_*` files are documented. **verified**
|
||||
([kernel thermal docs](https://docs.kernel.org/gpu/amdgpu/thermal.html)); the Arch Wiki adds
|
||||
`pp_od_clk_voltage`, `pp_power_profile_mode` (COMPUTE is profile 5) and warns that PowerPlay-table
|
||||
tools are "bugged on kernel side" for Navi 3x/4x. **verified**
|
||||
([Arch Wiki](https://wiki.archlinux.org/title/AMDGPU#Overclocking))
|
||||
- **Undervolt/power cap**: the tuning repo forces `manual`, writes `vo -85` + `c` to
|
||||
`pp_od_clk_voltage`, and writes watts to hwmon `power1_cap` from a systemd oneshot. No NixOS
|
||||
module exists. **community-reported**
|
||||
([tune_r9700.sh](https://github.com/stew675/r9700-setup/blob/HEAD/tune_r9700.sh))
|
||||
|
||||
## What this repository does, and what is still open
|
||||
|
||||
1. `hardware-amd-gpu.nix` sets `"amdgpu.runpm=0"` next to `iommu=pt`, over a `power/control=on` udev
|
||||
rule, which would leave the driver registered as a PX client. Revisit if a BIOS update defines
|
||||
the `_OFF`/`_ON` methods — or now that two cards may keep ATPX out of the picture (above).
|
||||
2. `iommu=pt` stays; `iommu=off` for P2P is a community claim with no AMD statement.
|
||||
3. `hardware.amdgpu.opencl.enable = true` is the shape the host uses — the module adds both `clr`
|
||||
and `clr.icd` — instead of a hand-written `hardware.graphics.extraPackages`.
|
||||
4. `rocmPackages.rocm-smi` is not wrapped for libdrm. It reads a healthy card completely; the
|
||||
`Fail to open libdrm_amdgpu.so` message only appeared while the card was wedged (see "ROCm
|
||||
userland in nixpkgs"). Expect `Card Series: N/A` until upstream names the R9700.
|
||||
5. `hardware.amdgpu.overdrive` stays off unless undervolting is wanted; if it is turned on, set
|
||||
`ppfeaturemask = "0xfff7ffff"` (nixpkgs#564601). Power-cap scripting stays out of Nix.
|
||||
`amdgpu.gpu_recovery=1` is set, because the driver default is auto/off except SRIOV.
|
||||
6. **Open:** set `HIP_VISIBLE_DEVICES` (and `ROCR_VISIBLE_DEVICES` where honoured) so the iGPU is
|
||||
never selected; for llama.cpp prefer `--split-mode layer` or an RPC/pipeline split over tensor
|
||||
parallelism, and build with `-DGGML_HIP=ON -DGPU_TARGETS=gfx1201`, no `HSA_OVERRIDE_GFX_VERSION`.
|
||||
7. Keep the x4 claim in [README.md](./README.md) and `hardware-amd-gpu.nix`: the chipset root port
|
||||
`80:1c.0` measures 16 GT/s x4 while both cards' own links are x16, which is why the engine's
|
||||
probe matters. Keep "split by layer, not tensor parallel" either way — it now rests on the RCCL
|
||||
TP=2 issue, not slot width.
|
||||
8. **Open:** make the ECC decision explicit: for GECC add `amdgpu.ras_enable=1` and confirm the boot
|
||||
log's `GECC is enabled` line; otherwise say so, because the kernel warns about it every boot.
|
||||
|
||||
## Not settled from primary sources
|
||||
|
||||
- The `PEG0.PEGP._OFF` → ATPX abort → SMU `-62` sequence is read from kernel source; no maintainer
|
||||
has confirmed the "ignored ATPX return value" step, and no upstream report of the chain was found.
|
||||
- No primary source states a minimum kernel for Navi 48, and none states that PCIe P2P between R9700
|
||||
cards is available — only the two-card report and llama.cpp's opt-out flag.
|
||||
- vLLM's official ROCm support for gfx1201 is undocumented; working configurations are community
|
||||
forks whose correctness is not independently verified.
|
||||
@@ -0,0 +1,139 @@
|
||||
{
|
||||
disko,
|
||||
myvars,
|
||||
lib,
|
||||
...
|
||||
}:
|
||||
#############################################################
|
||||
#
|
||||
# Aquamarine - headless AI inference server. Hardware, install and runbook:
|
||||
# ./README.md.
|
||||
#
|
||||
#############################################################
|
||||
let
|
||||
hostName = "aquamarine"; # Define your hostname.
|
||||
|
||||
inherit (myvars.networking) proxyGateway proxyGateway6 nameservers;
|
||||
inherit (myvars.networking.hostsAddr.${hostName}) ipv4;
|
||||
ipv4WithMask = "${ipv4}/24";
|
||||
in
|
||||
{
|
||||
imports = [
|
||||
disko.nixosModules.default
|
||||
# disks
|
||||
./disko-fs.nix
|
||||
|
||||
# hardware
|
||||
./hardware-configuration.nix
|
||||
./hardware-amd-gpu.nix
|
||||
|
||||
./preservation.nix
|
||||
./secureboot.nix
|
||||
];
|
||||
|
||||
# Zram consumes physical memory for compression, which deadlocks when the
|
||||
# model being loaded approaches the physical memory limit — exactly the
|
||||
# workload this host exists for. Use the disk swapfile that disko creates at
|
||||
# /swap instead, and keep zswap as a compressed cache in front of it so cold
|
||||
# anonymous pages stay in RAM rather than hitting the SSD.
|
||||
modules.zram.enable = false;
|
||||
boot.zswap = {
|
||||
enable = true;
|
||||
compressor = "zstd";
|
||||
# 10% of 96G; the module default 25% would compete with the page cache that
|
||||
# holds the model weights being read in.
|
||||
maxPoolPercent = 10;
|
||||
};
|
||||
|
||||
# Local btrfs snapshots of @persistent (SSH host keys, /etc/secureboot, the
|
||||
# NixOS state). Same layout as the other physical hosts; no off-host target.
|
||||
modules.btrbk.enable = true;
|
||||
|
||||
# smartd autodetects devices and logs to the journal; the smartctl exporter
|
||||
# (modules/nixos/base/monitoring.nix) is what feeds wear and media errors to
|
||||
# VictoriaMetrics.
|
||||
services.smartd.enable = true;
|
||||
|
||||
# The inference engine ships as an OCI image and is meant to run rootless, so
|
||||
# `podman` is what this host uses to build and start it from the `~/src` checkout
|
||||
# that ./preservation.nix keeps.
|
||||
virtualisation.podman.enable = true;
|
||||
|
||||
# systemd-journal - cap disk usage
|
||||
# https://www.freedesktop.org/software/systemd/man/latest/journald.conf.html
|
||||
services.journald.settings.Journal = {
|
||||
SystemMaxUse = "10G";
|
||||
RuntimeMaxUse = "256M";
|
||||
};
|
||||
|
||||
networking = {
|
||||
inherit hostName;
|
||||
|
||||
# off: this host uses networkd (below).
|
||||
networkmanager.enable = false;
|
||||
useDHCP = false;
|
||||
};
|
||||
|
||||
# suzi's fake-ip DNS hands out AAAA records whose IPv6 path does not work: a
|
||||
# `curl -6` to a registry returns 502 or times out, while `curl -4` answers in
|
||||
# 0.1 s. Clients that try the AAAA first (podman, git, pip) then hang on the
|
||||
# handshake, which is why docker.io, github.com and pypi.org looked blocked
|
||||
# here. These are RFC 6724's default precedences with the IPv4-mapped row
|
||||
# raised, so IPv4 sorts first; glibc drops its whole built-in table as soon as
|
||||
# the file lists any entry, hence spelling all of it out.
|
||||
#
|
||||
# The real fix belongs on that gateway: stop answering AAAA for proxied names
|
||||
# (`dns.ipv6: false` in mihomo; the module README says the same).
|
||||
networking.getaddrinfo.precedence = {
|
||||
"::1/128" = 50;
|
||||
"::/0" = 40;
|
||||
"2002::/16" = 30;
|
||||
"::/96" = 20;
|
||||
"::ffff:0:0/96" = 100;
|
||||
};
|
||||
|
||||
# A `boot.kernel.sysctl` that disabled IPv6 was removed on 2026-10-14: networkd
|
||||
# re-enables it on the wired link for the `DHCP = "ipv6"`/`IPv6AcceptRA` settings
|
||||
# below, so it never applied and the AAAA attempt already fails at once (WA-034).
|
||||
|
||||
networking.useNetworkd = true;
|
||||
systemd.network.enable = true;
|
||||
|
||||
# Predictable names follow the PCIe topology, so match the `en*` prefix instead of
|
||||
# a name; one wired port, so one link.
|
||||
systemd.network.networks."10-lan" = {
|
||||
matchConfig.Name = [ "en*" ];
|
||||
networkConfig = {
|
||||
Address = [
|
||||
ipv4WithMask
|
||||
];
|
||||
# DNS goes to the proxy gateway, which resolves through the proxy;
|
||||
# `nameservers` is the plain upstream-DNS alternative.
|
||||
# DNS = nameservers;
|
||||
DNS = [ proxyGateway ];
|
||||
DHCP = "ipv6"; # enable DHCPv6 only, so we can get a GUA.
|
||||
IPv6AcceptRA = true; # for Stateless IPv6 Autoconfiguration (SLAAC)
|
||||
LinkLocalAddressing = "ipv6";
|
||||
# The RTL8125BG flaps carrier on link re-training; without this networkd
|
||||
# tears the static address and route down and the host drops off the LAN.
|
||||
IgnoreCarrierLoss = "10s";
|
||||
};
|
||||
routes = [
|
||||
{
|
||||
Destination = "0.0.0.0/0";
|
||||
Gateway = proxyGateway;
|
||||
}
|
||||
{
|
||||
Destination = "::/0";
|
||||
Gateway = proxyGateway6;
|
||||
GatewayOnLink = true; # it's a gateway on local link.
|
||||
}
|
||||
];
|
||||
linkConfig.RequiredForOnline = "routable";
|
||||
};
|
||||
|
||||
# This value determines the NixOS release from which the default
|
||||
# settings for stateful data (file locations, database versions) are taken.
|
||||
# It is the release this host was installed on.
|
||||
system.stateVersion = "26.11";
|
||||
}
|
||||
@@ -0,0 +1,132 @@
|
||||
# Disko layout for idols-aquamarine's root disk: the Fanxiang S790 2TB that was
|
||||
# `ai`'s /data disk. Same shape as hosts/idols-ai/disko-fs.nix
|
||||
# (ESP + LUKS + btrfs subvolumes, ephemeral tmpfs root, preservation on
|
||||
# /persistent), so the two hosts share one layout and one restore procedure.
|
||||
#
|
||||
# The LUKS mapper is named `aqua-luks`, not `nixos-luks`: `ai` already owns that
|
||||
# name while this disk is formatted (./README.md). disko generates the matching
|
||||
# boot.initrd.luks entry.
|
||||
#
|
||||
# Destroy, format & mount as root (wipes the disk; run from ~/nix-config, and
|
||||
# check `lsblk` + the by-id path first — see hosts/idols-aquamarine/README.md):
|
||||
# sudo nix run github:nix-community/disko -- --mode destroy,format,mount hosts/idols-aquamarine/disko-fs.nix
|
||||
# Mount only (after the first format):
|
||||
# sudo nix run github:nix-community/disko -- --mode mount hosts/idols-aquamarine/disko-fs.nix
|
||||
{
|
||||
# Ephemeral root; preservation mounts /persistent for state.
|
||||
fileSystems."/persistent".neededForBoot = true;
|
||||
|
||||
disko.devices = {
|
||||
# Ephemeral root; relatime and mode=755 so systemd does not set 777.
|
||||
nodev."/" = {
|
||||
fsType = "tmpfs";
|
||||
mountOptions = [
|
||||
"nosuid" # Ignore setuid bit; no suid binaries live on the root tmpfs
|
||||
"nodev" # Ignore device files; nothing here needs to be a device node
|
||||
"size=4G"
|
||||
"relatime" # Update inode access times relative to modify/change time
|
||||
"mode=755"
|
||||
];
|
||||
};
|
||||
|
||||
disk.nixos-aquamarine = {
|
||||
type = "disk";
|
||||
device = "/dev/disk/by-id/nvme-Fanxiang_S790_2TB_FXS790254050582";
|
||||
content = {
|
||||
type = "gpt";
|
||||
partitions = {
|
||||
# EFI system partition; must stay unencrypted for UEFI to load the bootloader.
|
||||
ESP = {
|
||||
priority = 1;
|
||||
name = "ESP";
|
||||
start = "1M";
|
||||
end = "600M";
|
||||
type = "EF00"; # EF00 = ESP in GPT
|
||||
content = {
|
||||
type = "filesystem";
|
||||
format = "vfat";
|
||||
mountpoint = "/boot";
|
||||
mountOptions = [
|
||||
"fmask=0177" # File mask: 777-177=600 (owner rw-, group/others ---)
|
||||
"dmask=0077" # Directory mask: 777-077=700 (owner rwx, group/others ---)
|
||||
"noexec,nosuid,nodev" # Security: no execution, ignore setuid, no device nodes
|
||||
];
|
||||
};
|
||||
};
|
||||
# Root partition: LUKS encrypted, then btrfs with subvolumes.
|
||||
root = {
|
||||
size = "100%";
|
||||
content = {
|
||||
type = "luks";
|
||||
name = "aqua-luks"; # Mapper name; match boot.initrd.luks
|
||||
settings = {
|
||||
allowDiscards = true; # TRIM for SSDs; slightly less secure, better performance
|
||||
};
|
||||
# Add boot.initrd.luks.devices so initrd prompts for passphrase at boot
|
||||
initrdUnlock = true;
|
||||
# cryptsetup luksFormat options
|
||||
extraFormatArgs = [
|
||||
"--type luks2"
|
||||
"--cipher aes-xts-plain64"
|
||||
"--hash sha512"
|
||||
"--iter-time 5000"
|
||||
"--key-size 256"
|
||||
"--pbkdf argon2id"
|
||||
"--use-random" # Block until enough entropy from /dev/random
|
||||
];
|
||||
extraOpenArgs = [
|
||||
"--timeout 10"
|
||||
];
|
||||
content = {
|
||||
type = "btrfs";
|
||||
extraArgs = [ "-f" ]; # Force overwrite if filesystem already exists
|
||||
subvolumes = {
|
||||
# Top-level subvolume (id 5); used for btrfs send/receive and snapshots
|
||||
"/" = {
|
||||
mountpoint = "/btr_pool";
|
||||
mountOptions = [ "subvolid=5" ];
|
||||
};
|
||||
"@nix" = {
|
||||
mountpoint = "/nix";
|
||||
mountOptions = [
|
||||
"compress-force=zstd:1" # Save space and reduce I/O on SSD
|
||||
"noatime"
|
||||
];
|
||||
};
|
||||
"@persistent" = {
|
||||
mountpoint = "/persistent";
|
||||
mountOptions = [
|
||||
"compress-force=zstd:1"
|
||||
];
|
||||
};
|
||||
"@snapshots" = {
|
||||
mountpoint = "/snapshots";
|
||||
mountOptions = [
|
||||
"nosuid"
|
||||
"nodev"
|
||||
"compress-force=zstd:1"
|
||||
];
|
||||
};
|
||||
"@tmp" = {
|
||||
mountpoint = "/tmp";
|
||||
mountOptions = [
|
||||
"nosuid"
|
||||
"nodev"
|
||||
"noatime" # Temp files don't need access times; saves btrfs metadata writes
|
||||
"compress-force=zstd:1"
|
||||
];
|
||||
};
|
||||
# disko creates the 96G swapfile on this subvolume and adds swapDevices
|
||||
"@swap" = {
|
||||
mountpoint = "/swap";
|
||||
swap.swapfile.size = "96G";
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,84 @@
|
||||
{
|
||||
pkgs,
|
||||
myvars,
|
||||
...
|
||||
}:
|
||||
{
|
||||
# ===============================================================================================
|
||||
# 2x AMD Radeon AI PRO R9700 (RDNA4, gfx1201) for local inference
|
||||
# https://wiki.nixos.org/wiki/AMD_GPU
|
||||
#
|
||||
# The second full-length x16 slot is chipset-attached: its root port is Gen4 x4
|
||||
# while the card side reports x16, so the second card reads weights at ~7.1 GB/s
|
||||
# against 46.3 GB/s. Split models by layer, not tensor parallel — see ./README.md
|
||||
# and ./ROCM.md (ROCm support, measured links, RCCL TP=2 deadlock).
|
||||
# ===============================================================================================
|
||||
|
||||
boot.kernelModules = [ "amdgpu" ];
|
||||
|
||||
boot.kernelParams = [
|
||||
# ROCm: multi-GPU systems may hang in compute applications unless the IOMMU
|
||||
# runs in passthrough mode.
|
||||
# https://rocm.docs.amd.com/projects/install-on-linux/en/latest/reference/system-requirements.html
|
||||
"iommu=pt"
|
||||
|
||||
# This board's DSDT advertises ATPX power control (`\_SB.PC00.GFX0.ATPX`) but
|
||||
# never defines the `PEG0.PEGP._OFF`/`_ON` symbols its body calls, so the first
|
||||
# idle resume of the headless card leaves it in `runtime_status = error` until
|
||||
# reboot; amdgpu discards the ACPI error, so the failure is silent. This is the
|
||||
# only setting that avoids it, and it needs the iGPU kept for the console
|
||||
# fallback to trigger at all. Root cause, kernel-source trail and the field
|
||||
# comparison: ./ROCM.md (WA-032).
|
||||
"amdgpu.runpm=0"
|
||||
|
||||
# The driver's own default is -1, "auto, disabled except SRIOV", so on bare
|
||||
# metal a hung card stays hung until the next reboot. Nobody is here to
|
||||
# power-cycle an unattended inference host, so switch the recovery path on.
|
||||
"amdgpu.gpu_recovery=1"
|
||||
|
||||
# ASPM L0s/L1 on a link that carries PCIe traffic for a compute card buys
|
||||
# little and is a known source of link retraining and timeouts; this repo
|
||||
# already falls back to the same knob for NVMe APST problems (WA-008).
|
||||
"pcie_aspm=off"
|
||||
|
||||
# Keep the CPU out of the deepest C-states. Inference waits on short PCIe and
|
||||
# memory round-trips, and the exit latency from C6/C7 shows up as decode
|
||||
# jitter. Costs idle power: drop this line if that matters more than tail
|
||||
# latency.
|
||||
"processor.max_cstate=2"
|
||||
];
|
||||
|
||||
# amdgpu firmware.
|
||||
hardware.enableRedistributableFirmware = true;
|
||||
|
||||
hardware.graphics = {
|
||||
enable = true;
|
||||
};
|
||||
|
||||
# The ROCm OpenCL ICD, which is also what HIP consumers resolve for compute.
|
||||
hardware.amdgpu.opencl.enable = true;
|
||||
|
||||
# /dev/kfd is world-writable under nixpkgs' default udev rule, but
|
||||
# /dev/dri/renderD* is `render` 0660 and this headless host has no logged-in
|
||||
# seat to hand out ACLs, so the inference user needs the group.
|
||||
users.users.${myvars.username}.extraGroups = [ "render" ];
|
||||
|
||||
environment.systemPackages = [
|
||||
pkgs.rocmPackages.rocm-smi
|
||||
pkgs.rocmPackages.rocminfo
|
||||
pkgs.nvtopPackages.full
|
||||
];
|
||||
|
||||
# GPU panel in btop, and utilization/VRAM in node_exporter for the shared
|
||||
# VictoriaMetrics scrape — the same shape as the Radeon iGPU hosts.
|
||||
# perfmon: CAP_PERFMON is what btop needs for the i915 iGPU.
|
||||
modules.btop = {
|
||||
package = pkgs.btop-rocm;
|
||||
perfmon = true;
|
||||
};
|
||||
# `enabledCollectors` is a list option, so this merges with the base module's
|
||||
# `[ "systemd" "logind" ]` instead of replacing it: only name the collector
|
||||
# this host adds. Repeating the base entries passes `--collector.systemd`
|
||||
# twice, which node_exporter rejects, and the unit restart-loops.
|
||||
services.prometheus.exporters.node.enabledCollectors = [ "drm" ];
|
||||
}
|
||||
@@ -0,0 +1,73 @@
|
||||
# Hand-adapted from the `nixos-generate-config --show-hardware-config` scan that
|
||||
# `ai` captured on this same Colorful CVN Z890 ARK FROZEN V20 board, with the
|
||||
# KINGBANK-specific parts left behind. The
|
||||
# mounts themselves are declared in ./disko-fs.nix, so nothing here names a
|
||||
# disk. Regenerate this file on the target (`nixos-generate-config --root /mnt`)
|
||||
# if the hardware scan ever differs.
|
||||
{
|
||||
config,
|
||||
lib,
|
||||
pkgs,
|
||||
modulesPath,
|
||||
...
|
||||
}:
|
||||
{
|
||||
imports = [
|
||||
(modulesPath + "/installer/scan/not-detected.nix")
|
||||
];
|
||||
|
||||
boot.kernelParams = [
|
||||
# Carried over from `ai`, where it caps NVMe APST wake latency for the
|
||||
# DRAM-less KINGBANK KP260 (WA-008 in WORKAROUNDS.md). This disk (Fanxiang
|
||||
# S790) has shown no APST problem; the cap stays as idle-power insurance.
|
||||
"nvme_core.default_ps_max_latency_us=100000"
|
||||
];
|
||||
|
||||
boot.loader.efi.canTouchEfiVariables = true;
|
||||
boot.loader.efi.efiSysMountPoint = "/boot";
|
||||
|
||||
# The nixpkgs default (`linuxPackages`) is the stable/LTS line, which is what a
|
||||
# server should run; `linuxPackages_latest` is for chasing new hardware. The
|
||||
# R9700 (Navi 48) only needs the 6.14 AMD validates for it, so the default is
|
||||
# far ahead of the requirement. See ./ROCM.md.
|
||||
# https://github.com/nixpkgs/nixpkgs/blob/nixos-unstable/pkgs/top-level/linux-kernels.nix
|
||||
boot.kernelPackages = pkgs.linuxPackages;
|
||||
|
||||
boot.initrd.availableKernelModules = [
|
||||
"xhci_pci"
|
||||
"ahci"
|
||||
"nvme"
|
||||
"usbhid"
|
||||
"usb_storage"
|
||||
"sd_mod"
|
||||
];
|
||||
# clear /tmp on boot to get a stateless /tmp directory.
|
||||
boot.tmp.cleanOnBoot = true;
|
||||
|
||||
# Enable binfmt emulation of aarch64-linux, this is required for cross compilation.
|
||||
boot.binfmt.emulatedSystems = [
|
||||
"aarch64-linux"
|
||||
"riscv64-linux"
|
||||
];
|
||||
# This enables the kernel to preload the emulator binaries when the binfmt registrations are added,
|
||||
# obviating the need to make the emulator binaries available inside chroots and chroot-like sandboxes.
|
||||
boot.binfmt.preferStaticEmulators = true; # required to work with podman
|
||||
|
||||
# supported file systems, so we can mount any removable disks with these filesystems
|
||||
boot.supportedFilesystems = [
|
||||
"ext4"
|
||||
"btrfs"
|
||||
"xfs"
|
||||
"ntfs"
|
||||
"fat"
|
||||
"vfat"
|
||||
"exfat"
|
||||
];
|
||||
|
||||
# LUKS initrd, all fileSystems (/, /boot, /btr_pool, /nix, /persistent, /snapshots, /tmp, /swap)
|
||||
# and swap (including /swap/swapfile bind and swapDevices) are managed by disko (disko-fs.nix).
|
||||
|
||||
nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux";
|
||||
powerManagement.cpuFreqGovernor = lib.mkDefault "performance"; # ondemand / powersave / performance
|
||||
hardware.cpu.intel.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware;
|
||||
}
|
||||
@@ -0,0 +1,110 @@
|
||||
{
|
||||
preservation,
|
||||
lib,
|
||||
pkgs,
|
||||
myvars,
|
||||
...
|
||||
}:
|
||||
let
|
||||
inherit (myvars) username;
|
||||
in
|
||||
{
|
||||
imports = [
|
||||
preservation.nixosModules.default
|
||||
];
|
||||
|
||||
preservation.enable = true;
|
||||
# preservation requires a systemd initrd.
|
||||
boot.initrd.systemd.enable = true;
|
||||
|
||||
environment.systemPackages = [
|
||||
# Whole-filesystem disk usage (`-x` stays on one filesystem):
|
||||
# sudo gdu -n -x / # non-interactive, plain text
|
||||
pkgs.gdu
|
||||
];
|
||||
|
||||
# NOTE: preservation only mounts the directory/file list below to /persistent
|
||||
# If the directory/file already exists in the root filesystem you should
|
||||
# move those files/directories to /persistent first!
|
||||
preservation.preserveAt."/persistent" = {
|
||||
directories = [
|
||||
"/etc/ssh"
|
||||
"/etc/nix/inputs"
|
||||
"/etc/secureboot" # lanzaboote - secure boot (see ./secureboot.nix)
|
||||
|
||||
"/var/log"
|
||||
"/var/lib"
|
||||
"/var/cache"
|
||||
# Re-creatable scratch, but it must not consume the root tmpfs (RAM), so
|
||||
# it is bind-mounted onto the persistent volume and wiped at boot below.
|
||||
# The mode must stay 1777 to match that `D!` rule.
|
||||
{
|
||||
directory = "/var/tmp";
|
||||
mode = "1777";
|
||||
mountOptions = [
|
||||
"nosuid"
|
||||
"nodev"
|
||||
];
|
||||
}
|
||||
];
|
||||
files = [
|
||||
# auto-generated machine ID
|
||||
{
|
||||
file = "/etc/machine-id";
|
||||
inInitrd = true;
|
||||
}
|
||||
];
|
||||
|
||||
# the following directories will be passed to /persistent/home/$USER
|
||||
users.${username} = {
|
||||
directories = [
|
||||
# Rootless Podman's image and layer store. On the root tmpfs the 4 GiB root
|
||||
# fills up during the first build, so it lives on the persistent volume
|
||||
# like it does on `ai`.
|
||||
".local/share/containers"
|
||||
"codes"
|
||||
# Third-party source checkouts, the same layout `ai` uses: Strata, the
|
||||
# inference engine whose image this host builds, lives at ~/src/Strata.
|
||||
"src"
|
||||
"nix-config"
|
||||
"tmp"
|
||||
# Model weights live here. The root filesystem is tmpfs, so anything
|
||||
# not listed is gone on the next boot; weights are re-downloadable, but
|
||||
# re-downloading tens of GB is not a boot step.
|
||||
"models"
|
||||
];
|
||||
};
|
||||
};
|
||||
|
||||
# `.config`/`.local` are not immediate parents of any persisted path here, so
|
||||
# tmpfiles would create them root:root 0755 and the user could not write
|
||||
# under them. Same fix as hosts/idols-ai/preservation.nix.
|
||||
systemd.tmpfiles.settings.preservation = {
|
||||
"/home/${username}/.config".d = {
|
||||
user = username;
|
||||
group = lib.mkForce username;
|
||||
mode = lib.mkForce "0750";
|
||||
};
|
||||
"/home/${username}/.local".d = {
|
||||
user = username;
|
||||
group = lib.mkForce username;
|
||||
mode = lib.mkForce "0750";
|
||||
};
|
||||
"/home/${username}/.local/share".d = {
|
||||
user = username;
|
||||
group = lib.mkForce username;
|
||||
mode = lib.mkForce "0750";
|
||||
};
|
||||
};
|
||||
|
||||
systemd.tmpfiles.rules = [
|
||||
# /var/tmp only accumulates between reboots (mirrors `boot.tmp.cleanOnBoot`
|
||||
# for /tmp). Crash dumps can hold secrets copied out of process memory.
|
||||
"D! /var/tmp 1777 root root"
|
||||
"D! /var/lib/systemd/coredump 0755 root root"
|
||||
];
|
||||
|
||||
# Mask the commit service: /etc/machine-id is persisted by preservation
|
||||
# (see the `files` entry above), so there is no transient ID to commit.
|
||||
systemd.suppressedSystemUnits = [ "systemd-machine-id-commit.service" ];
|
||||
}
|
||||
@@ -0,0 +1,30 @@
|
||||
{
|
||||
pkgs,
|
||||
lib,
|
||||
lanzaboote,
|
||||
...
|
||||
}:
|
||||
{
|
||||
# Secure Boot here reuses the PKI already enrolled on the Z890 board, carried over with the disk
|
||||
# (./README.md step 3). lanzaboote signs with `pkiBundle` below, so a fresh key set must end up in
|
||||
# /etc/secureboot — `sbctl create-keys` writes to its own keydir — or the firmware trusts a key the
|
||||
# UKI was not signed with. New keys also change PCR7, which invalidates a TPM2 LUKS enrollment made
|
||||
# against the old ones (./README.md step 4).
|
||||
imports = [
|
||||
lanzaboote.nixosModules.lanzaboote
|
||||
];
|
||||
|
||||
environment.systemPackages = [
|
||||
# For debugging and troubleshooting Secure Boot.
|
||||
pkgs.sbctl
|
||||
];
|
||||
|
||||
# Lanzaboote replaces the systemd-boot module. The generated
|
||||
# hardware-configuration.nix sets this to true, so force it off.
|
||||
boot.loader.systemd-boot.enable = lib.mkForce false;
|
||||
|
||||
boot.lanzaboote = {
|
||||
enable = true;
|
||||
pkiBundle = "/etc/secureboot";
|
||||
};
|
||||
}
|
||||
@@ -48,8 +48,8 @@ nix run github:nix-community/disko -- --mode destroy,format,mount ../hosts/idols
|
||||
# Mount only (e.g. after first format, without wiping):
|
||||
# nix run github:nix-community/disko -- --mode mount ../hosts/idols-ai/disko-fs.nix
|
||||
|
||||
# the separate data disk (mounts at /data):
|
||||
nix run github:nix-community/disko -- --mode destroy,format,mount ../hosts/idols-ai/disko-fs-data.nix
|
||||
# Other hosts keep their own layout file. The inference server's root disk is installed from a
|
||||
# running host, not from this ISO flow: see ../hosts/idols-aquamarine/README.md for the order.
|
||||
|
||||
# setup the automatic unlock via the tpm2 chip
|
||||
# find the LUKS partition with: lsblk -f
|
||||
|
||||
+2
-2
@@ -48,9 +48,9 @@ them. Others name specific hosts — or a name prefix, like `k3s-api-endpoint`,
|
||||
`k3s-test-1-*` configuration — and only cover a new host if you add it.
|
||||
|
||||
| Kind | Tests |
|
||||
| ------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
||||
| ------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
||||
| Every configuration | `hostname`, `kernel`, `nix-system-features`, `security-apparmor`, `security-container-groups`, `security-exporters`, `security-firewall`, `security-k3s-kubeconfig`, `security-k3s-pod-firewall`, `security-kernel`, `security-ssh-x11` |
|
||||
| Named hosts only | `btrbk`, `home-manager`, `home-manager-xdg`, `computer-use-headless`, `k3s-api-endpoint`, `k3s-node-home-manager`, `idols-ai-gpu`, `idols-ai-voice-input`, `security-postgres-auth`, `youko-metrics`, `ups-metrics`, `shoukei-logind` |
|
||||
| Named hosts only | `btrbk`, `home-manager`, `home-manager-xdg`, `computer-use-headless`, `k3s-api-endpoint`, `k3s-node-home-manager`, `idols-ai-gpu`, `idols-ai-voice-input`, `idols-aquamarine`, `security-postgres-auth`, `youko-metrics`, `ups-metrics`, `shoukei-logind` |
|
||||
|
||||
`hostname` expects each configuration's `networking.hostName` to equal its name. Niri desktop
|
||||
configurations are the exception (`ai-niri` → `ai`, `shoukei-niri` → `shoukei`) and are listed in
|
||||
|
||||
@@ -0,0 +1,50 @@
|
||||
{
|
||||
# NOTE: the args not used in this file CAN NOT be removed!
|
||||
# because haumea pass argument lazily,
|
||||
# and these arguments are used in the functions like `mylib.nixosSystem`, `mylib.colmenaSystem`, etc.
|
||||
inputs,
|
||||
lib,
|
||||
mylib,
|
||||
myvars,
|
||||
system,
|
||||
genSpecialArgs,
|
||||
...
|
||||
}@args:
|
||||
let
|
||||
# 星野 愛久愛海, Hoshino Akuamarin
|
||||
name = "aquamarine";
|
||||
tags = [
|
||||
name
|
||||
"homelab-ai"
|
||||
];
|
||||
ssh-user = "root";
|
||||
|
||||
modules = {
|
||||
nixos-modules =
|
||||
(map mylib.relativeToRoot [
|
||||
# common
|
||||
"modules/nixos/server/server.nix"
|
||||
# host specific
|
||||
"hosts/idols-${name}"
|
||||
])
|
||||
++ [
|
||||
# No `modules.secrets.server.*` group yet: the host would fail activation
|
||||
# until its new host key is a recipient in nix-secrets. Enabling one is
|
||||
# the last step in hosts/idols-aquamarine/README.md.
|
||||
];
|
||||
};
|
||||
|
||||
systemArgs = modules // args;
|
||||
in
|
||||
{
|
||||
nixosConfigurations.${name} = mylib.nixosSystem systemArgs;
|
||||
|
||||
colmena.${name} = mylib.colmenaSystem (
|
||||
systemArgs
|
||||
// {
|
||||
inherit tags ssh-user;
|
||||
}
|
||||
);
|
||||
|
||||
packages.${name} = inputs.self.nixosConfigurations.${name}.config.system.build.images.iso;
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
{
|
||||
hostNameIsBare = true;
|
||||
amdgpuDriverLoaded = true;
|
||||
iommuPassthrough = true;
|
||||
runtimePmDisabled = true;
|
||||
gpuRecoveryEnabled = true;
|
||||
aspmDisabled = true;
|
||||
cStatesCapped = true;
|
||||
prefersIpv4 = true;
|
||||
sourceCheckoutsPersistent = true;
|
||||
rootlessPodmanEnabled = true;
|
||||
stableKernel = true;
|
||||
btopShowsRadeon = true;
|
||||
btopReadsIgpu = true;
|
||||
gpuMetricsExported = true;
|
||||
collectorFlagsUnique = true;
|
||||
inferenceUserCanRender = true;
|
||||
noHomeManager = true;
|
||||
staticAddressMatchesWiredInterface = true;
|
||||
rootIsTmpfs = true;
|
||||
stateIsPersistent = true;
|
||||
luksMapperNamed = true;
|
||||
}
|
||||
@@ -0,0 +1,67 @@
|
||||
{
|
||||
myvars,
|
||||
outputs,
|
||||
lib,
|
||||
...
|
||||
}:
|
||||
let
|
||||
eval = outputs.nixosConfigurations.aquamarine;
|
||||
cfg = eval.config;
|
||||
inherit (myvars.networking.hostsAddr.aquamarine) ipv4;
|
||||
in
|
||||
{
|
||||
# A server configuration is named after the bare hostname (no `-niri`).
|
||||
hostNameIsBare = cfg.networking.hostName == "aquamarine";
|
||||
|
||||
# The two Radeon AI PRO R9700 cards: driver and the IOMMU mode ROCm needs for multi-GPU.
|
||||
amdgpuDriverLoaded = builtins.elem "amdgpu" cfg.boot.kernelModules;
|
||||
iommuPassthrough = builtins.elem "iommu=pt" cfg.boot.kernelParams;
|
||||
# This board's ACPI firmware cannot power the dGPU back on, and amdgpu throws
|
||||
# the error away, so the card dies on its first idle resume (./ROCM.md, WA-032).
|
||||
runtimePmDisabled = builtins.elem "amdgpu.runpm=0" cfg.boot.kernelParams;
|
||||
|
||||
# An unattended host recovers a hung card itself instead of waiting for a reboot.
|
||||
gpuRecoveryEnabled = builtins.elem "amdgpu.gpu_recovery=1" cfg.boot.kernelParams;
|
||||
aspmDisabled = builtins.elem "pcie_aspm=off" cfg.boot.kernelParams;
|
||||
cStatesCapped = builtins.elem "processor.max_cstate=2" cfg.boot.kernelParams;
|
||||
|
||||
# suzi's fake-ip DNS returns AAAA records with no working IPv6 path, so the
|
||||
# generated gai.conf has to sort IPv4 first (see ./default.nix).
|
||||
prefersIpv4 = lib.hasInfix "precedence ::ffff:0:0/96 100" cfg.environment.etc."gai.conf".text;
|
||||
|
||||
# The Strata checkout lives under ~/src, which has to survive the tmpfs root
|
||||
# like ~/codes does.
|
||||
sourceCheckoutsPersistent = builtins.any (
|
||||
d: d.directory == "/home/${myvars.username}/src"
|
||||
) cfg.preservation.preserveAt."/persistent".users.${myvars.username}.directories;
|
||||
rootlessPodmanEnabled = cfg.virtualisation.podman.enable;
|
||||
|
||||
# A server tracks the nixpkgs default (stable/LTS) kernel, not `-latest`.
|
||||
stableKernel = cfg.boot.kernelPackages.kernel.version == eval.pkgs.linuxPackages.kernel.version;
|
||||
btopShowsRadeon = cfg.modules.btop.package == eval.pkgs.btop-rocm;
|
||||
# CAP_PERFMON inside the wrapper is what the i915 iGPU needs.
|
||||
btopReadsIgpu = cfg.modules.btop.perfmon;
|
||||
gpuMetricsExported = builtins.elem "drm" cfg.services.prometheus.exporters.node.enabledCollectors;
|
||||
# node_exporter rejects a repeated flag, and this list merges with the base
|
||||
# module's entries, so a duplicate restart-loops the unit.
|
||||
collectorFlagsUnique =
|
||||
builtins.length (cfg.services.prometheus.exporters.node.enabledCollectors)
|
||||
== builtins.length (lib.unique cfg.services.prometheus.exporters.node.enabledCollectors);
|
||||
inferenceUserCanRender = builtins.elem "render" cfg.users.users.${myvars.username}.extraGroups;
|
||||
|
||||
# Headless: no desktop session, no Home Manager.
|
||||
noHomeManager = !cfg ? home-manager.users;
|
||||
|
||||
# The board renames the wired NIC with the PCIe topology, so networkd matches the
|
||||
# `en*` prefix; a name-based match leaves the host without an address.
|
||||
staticAddressMatchesWiredInterface =
|
||||
builtins.elem "${ipv4}/24" cfg.systemd.network.networks."10-lan".networkConfig.Address
|
||||
&& cfg.systemd.network.networks."10-lan".matchConfig.Name == [ "en*" ];
|
||||
|
||||
# Ephemeral root with the persistent volume required at boot, as on `ai`.
|
||||
rootIsTmpfs = cfg.fileSystems."/".fsType or null == "tmpfs";
|
||||
stateIsPersistent = cfg.fileSystems."/persistent".neededForBoot or false;
|
||||
|
||||
# Install step 4 of ./README.md depends on this mapper name staying put.
|
||||
luksMapperNamed = builtins.attrNames cfg.boot.initrd.luks.devices == [ "aqua-luks" ];
|
||||
}
|
||||
+6
-3
@@ -50,8 +50,9 @@ rec {
|
||||
ipv6 = "fe80::10"; # Link-local Address
|
||||
};
|
||||
aquamarine = {
|
||||
# VM (retired)
|
||||
iface = "enp2s0";
|
||||
# Physical AI inference server. Reuses the retired VM's address, so check
|
||||
# that the router still reserves .101 for this machine. No `iface`: the host
|
||||
# matches the wired interface by its `en*` prefix (hosts/idols-aquamarine/).
|
||||
ipv4 = "192.168.5.101";
|
||||
};
|
||||
ruby = {
|
||||
@@ -202,7 +203,9 @@ rec {
|
||||
{
|
||||
# Define the root user's host key for remote builders, so that nix can verify all the remote builders
|
||||
|
||||
aquamarine.publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIEOXFhFu9Duzp6ZBE288gDZ6VLrNaeWL4kDrFUh9Neic root@aquamarine";
|
||||
# Reinstalled on 2026-10-13; this is the host key its first boot
|
||||
# generated (hosts/idols-aquamarine/README.md).
|
||||
aquamarine.publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHeZ1eTvzcoOOSmRCto/v9hKfirzaGh6/7bl5I2ly3Hj root@aquamarine";
|
||||
# the restic backup server
|
||||
youko.publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIEs8b3CmxM+utXZVKvJBssXBupBXvvzjHP40HcY7PSyZ root@youko";
|
||||
# ruby.publicKey = "";
|
||||
|
||||
Reference in new issue
Block a user