diff --git a/.typos.toml b/.typos.toml index b7d82659..c64a9a2d 100644 --- a/.typos.toml +++ b/.typos.toml @@ -63,4 +63,8 @@ extend-ignore-re = [ # checker wants `also`, and the pre-commit hook rewrites it. "als", "ALS", + + # ROCm's `HSA_OVERRIDE_GFX_VERSION` (Heterogeneous System Architecture); the + # dictionary wants "HAS". + "HSA", ] diff --git a/AGENTS.md b/AGENTS.md index 43faa9c0..4352eb6e 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -99,9 +99,9 @@ introduces one, and re-evaluate entries when their `Revisit` trigger comes up. password prompt; `just darwin-rollback` switches the macOS generation on the machine you are on, and a bare `sudo systemctl restart ` activates a running service the same way. The user runs these, not an agent. -- `just shoryu`, `just shushou`, `just youko`, `just ruby`, `just kana`, `just lab`, - `just k3s-test`, and `just col ` activate systems through Colmena. Use the narrower recipe - that matches the intended host scope. +- `just shoryu`, `just shushou`, `just youko`, `just aquamarine`, `just ruby`, `just kana`, + `just lab`, `just k3s-test`, and `just col ` activate systems through Colmena. Use the + narrower recipe that matches the intended host scope. - `just microvm-deploy ` installs one MicroVM guest's runner on the physical host and restarts its unit there. A later host activation re-points the guest at the runner baked into the host's system; see WA-026 in [WORKAROUNDS.md](./WORKAROUNDS.md) before deploying. Deploy diff --git a/WORKAROUNDS.md b/WORKAROUNDS.md index b4ea908e..0a332e91 100644 --- a/WORKAROUNDS.md +++ b/WORKAROUNDS.md @@ -33,20 +33,22 @@ A version or revision pinned for a reason that can go away. A setting that works around a bug or a hardware quirk. -| ID | What & where | Why | Removal condition | Added | Revisit | Status | -| ------ | --------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------- | --------------------------------------------------------------------------------------------------- | ------ | -| WA-008 | Relaxed NVMe APST (`nvme_core.default_ps_max_latency_us=100000`) — `hosts/idols-ai/hardware-configuration.nix` | the KINGBANK KP260 (DRAM-less QLC) froze on slow APST wake-ups | Drop the override once kernel/firmware fixes or replacement hardware make default APST reliable | 2026-10-03 | watch NVMe timeout/reset/AER in the journal | active | -| WA-009 | USB-SATA bridge mitigations (`usbcore.autosuspend=-1`, `usb-storage.delay_use=10`) — `hosts/12kingdoms-youko/default.nix` | flaky JMicron JMS567 resets with UDMA-CRC and I/O errors | Drop once the bridge/controller is replaced; the signal is smartctl CRC/media errors | 2026-09-21 | on hardware change or CRC errors | active | -| WA-010 | Insta360 Link USB reset on S3 resume — `hosts/idols-ai/default.nix` | the camera stays enumerated with a stalled UVC endpoint after resume | Drop after the Insta360 firmware or the kernel `uvcvideo` fixes it | 2026-08-01 | on kernel or firmware change | active | -| WA-011 | Kernel module blacklist (`esp4`, `esp6`, `rxrpc`) — `hardening/kernel-hardening.nix` | attack surface for the Dirty Frag LPE (CVE-2026-43284 / CVE-2026-43500) | Kept as defence in depth; drop only if something needs those modules | 2026-10-03 | rarely | active | -| WA-017 | keyd right-hand modifier passthrough — `modules/nixos/desktop/peripherals.nix` | keyd folds right Ctrl/Shift/Meta into their left keys | Drop once keyd or fcitx5 keeps right-hand modifiers distinct | 2026-10-05 | on keyd bump | active | -| WA-018 | user-owned `~/.ssh/config` + disabled systemd ssh-proxy include — `home/base/tui/ssh.nix`, `modules/nixos/base/ssh.nix` | bwrap exposes root-owned store files as `nobody` | Drop once sandboxes preserve store ownership | 2026-10-06 | when sandbox layout changes | active | -| WA-019 | programs.steam.extraCompatPackages = [ pkgs.dwproton-bin pkgs.proton-ge-bin ] — modules/nixos/desktop/gaming.nix | ACE games fail on GE/mainline Proton, work with DW-Proton | Drop once mainline or GE-Proton runs these games (GE-Proton#433) | 2026-10-07 | on Proton/GE bump | active | -| WA-024 | Wi-Fi power saving disabled (`networking.networkmanager.wifi.powersave = false`) — `hosts/12kingdoms-shoukei/apple-silicon.nix` | On this BCM4378/brcmfmac laptop, powersave is a common source of intermittent ASSOC-REJECT / authentication-timeout failures; NetworkManager then mistakes the failure for a bad key, re-prompts, and the client (Noctalia, nmtui, nmcli) saves the answer as a duplicate ` 1` profile | Drop once the kernel carries the brcmfmac `port_authorized` fix (WA-025) and no association failures remain | 2026-10-07 | watch `journalctl -u wpa_supplicant` for ASSOC-REJECT / authentication timeout | active | -| WA-026 | guest deploys run `installOnHost` then `systemctl restart microvm@` on the physical host — `Justfile`, `lib/genMicrovmGuestModule.nix` | the sshSwitch step runs `nix-store --load-db` against the guest's read-only virtiofs `/nix/store` and dies on `cannot remount "/nix/store" writable` (nix 2.35) | Drop the manual path once microvm.nix or nix handles a read-only store switch. A host activation re-links `/var/lib/microvms//current` to the runner baked into the host's system, so switch the host after installing a guest from a newer tree | 2026-10-09 | on microvm.nix or nix bump | active | -| WA-028 | GitHub SSH pinned to IPv4 (`AddressFamily = "inet"`) — `home/base/tui/ssh.nix` | a v6 fake AAAA for `ssh.github.com` from the `2001:2::/64` pool (WA-029) is not proven on the 443 SSH tunnel yet | Drop once the v6 fake-ip pool is proven for GitHub SSH | 2026-10-10 | on a mihomo bump or a v6 pool change | active | -| WA-029 | mihomo `fake-ip-range6` is `2001:2::/64`, not the shipped ULA default — `modules/nixos/desktop/networking/mihomo/generate.nu` | the default `fdfe:dcba:9876::/64` sits in `fc00::/7`, which Local Network Access calls local, so a public page loaded over the v4 fake pool and pulling a dual-stack subresource prompts for local-network access (Chrome 154, Firefox). The same range also stalled ssh: a session to a fake AAAA authenticated and then never closed, on both `gvisor` and `mips`. `2001:2::/48` is RFC 5180 benchmarking -- the v6 twin of the `198.18.0.0/15` v4 default, absent from that table -- and both symptoms went with the move | Go back to the upstream default once mihomo ships a pool outside the LNA non-public table; the ULA default brings the ssh stall back | 2026-10-09 | on a browser or mihomo bump -- that table already grew to cover `2001:db8::/32` and `198.18.0.0/15` | active | -| WA-030 | mihomo unit sets `SKIP_SYSTEM_IPV6_CHECK=1` — `modules/nixos/desktop/networking/mihomo/default.nix` | mihomo checks at startup whether another NIC already holds an IPv6 address and, finding none, skips the TUN's v6 half -- but `strict-route` still installs its v6 blackhole (`ip -6 rule add priority 9000 unreachable`). On a cold boot that rule lands before networkd's Router Solicitation is answered, so the host can never send RS/NS, never gets a GUA, and mihomo never re-checks: self-locking until the core restarts. The env var forces the TUN v6 address up, which installs mihomo's normal v6 rules whose `lookup main suppress_prefixlength 0` keeps on-link neighbor discovery and the RA default route off the TUN | Drop once mihomo either re-runs the IPv6 check or skips the v6 strict-route blackhole when the v6 TUN is disabled | 2026-10-10 | on a mihomo bump | active | +| ID | What & where | Why | Removal condition | Added | Revisit | Status | +| ------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------- | --------------------------------------------------------------------------------------------------- | ------ | +| WA-008 | Relaxed NVMe APST (`nvme_core.default_ps_max_latency_us=100000`) — `hosts/idols-ai/hardware-configuration.nix`, `hosts/idols-aquamarine/hardware-configuration.nix` | the KINGBANK KP260 (DRAM-less QLC) froze on slow APST wake-ups | Drop the override once kernel/firmware fixes or replacement hardware make default APST reliable | 2026-10-03 | watch NVMe timeout/reset/AER in the journal | active | +| WA-009 | USB-SATA bridge mitigations (`usbcore.autosuspend=-1`, `usb-storage.delay_use=10`) — `hosts/12kingdoms-youko/default.nix` | flaky JMicron JMS567 resets with UDMA-CRC and I/O errors | Drop once the bridge/controller is replaced; the signal is smartctl CRC/media errors | 2026-09-21 | on hardware change or CRC errors | active | +| WA-010 | Insta360 Link USB reset on S3 resume — `hosts/idols-ai/default.nix` | the camera stays enumerated with a stalled UVC endpoint after resume | Drop after the Insta360 firmware or the kernel `uvcvideo` fixes it | 2026-08-01 | on kernel or firmware change | active | +| WA-011 | Kernel module blacklist (`esp4`, `esp6`, `rxrpc`) — `hardening/kernel-hardening.nix` | attack surface for the Dirty Frag LPE (CVE-2026-43284 / CVE-2026-43500) | Kept as defence in depth; drop only if something needs those modules | 2026-10-03 | rarely | active | +| WA-017 | keyd right-hand modifier passthrough — `modules/nixos/desktop/peripherals.nix` | keyd folds right Ctrl/Shift/Meta into their left keys | Drop once keyd or fcitx5 keeps right-hand modifiers distinct | 2026-10-05 | on keyd bump | active | +| WA-018 | user-owned `~/.ssh/config` + disabled systemd ssh-proxy include — `home/base/tui/ssh.nix`, `modules/nixos/base/ssh.nix` | bwrap exposes root-owned store files as `nobody` | Drop once sandboxes preserve store ownership | 2026-10-06 | when sandbox layout changes | active | +| WA-019 | programs.steam.extraCompatPackages = [ pkgs.dwproton-bin pkgs.proton-ge-bin ] — modules/nixos/desktop/gaming.nix | ACE games fail on GE/mainline Proton, work with DW-Proton | Drop once mainline or GE-Proton runs these games (GE-Proton#433) | 2026-10-07 | on Proton/GE bump | active | +| WA-024 | Wi-Fi power saving disabled (`networking.networkmanager.wifi.powersave = false`) — `hosts/12kingdoms-shoukei/apple-silicon.nix` | On this BCM4378/brcmfmac laptop, powersave is a common source of intermittent ASSOC-REJECT / authentication-timeout failures; NetworkManager then mistakes the failure for a bad key, re-prompts, and the client (Noctalia, nmtui, nmcli) saves the answer as a duplicate ` 1` profile | Drop once the kernel carries the brcmfmac `port_authorized` fix (WA-025) and no association failures remain | 2026-10-07 | watch `journalctl -u wpa_supplicant` for ASSOC-REJECT / authentication timeout | active | +| WA-026 | guest deploys run `installOnHost` then `systemctl restart microvm@` on the physical host — `Justfile`, `lib/genMicrovmGuestModule.nix` | the sshSwitch step runs `nix-store --load-db` against the guest's read-only virtiofs `/nix/store` and dies on `cannot remount "/nix/store" writable` (nix 2.35) | Drop the manual path once microvm.nix or nix handles a read-only store switch. A host activation re-links `/var/lib/microvms//current` to the runner baked into the host's system, so switch the host after installing a guest from a newer tree | 2026-10-09 | on microvm.nix or nix bump | active | +| WA-028 | GitHub SSH pinned to IPv4 (`AddressFamily = "inet"`) — `home/base/tui/ssh.nix` | a v6 fake AAAA for `ssh.github.com` from the `2001:2::/64` pool (WA-029) is not proven on the 443 SSH tunnel yet | Drop once the v6 fake-ip pool is proven for GitHub SSH | 2026-10-10 | on a mihomo bump or a v6 pool change | active | +| WA-029 | mihomo `fake-ip-range6` is `2001:2::/64`, not the shipped ULA default — `modules/nixos/desktop/networking/mihomo/generate.nu` | the default `fdfe:dcba:9876::/64` sits in `fc00::/7`, which Local Network Access calls local, so a public page loaded over the v4 fake pool and pulling a dual-stack subresource prompts for local-network access (Chrome 154, Firefox). The same range also stalled ssh: a session to a fake AAAA authenticated and then never closed, on both `gvisor` and `mips`. `2001:2::/48` is RFC 5180 benchmarking -- the v6 twin of the `198.18.0.0/15` v4 default, absent from that table -- and both symptoms went with the move | Go back to the upstream default once mihomo ships a pool outside the LNA non-public table; the ULA default brings the ssh stall back | 2026-10-09 | on a browser or mihomo bump -- that table already grew to cover `2001:db8::/32` and `198.18.0.0/15` | active | +| WA-030 | mihomo unit sets `SKIP_SYSTEM_IPV6_CHECK=1` — `modules/nixos/desktop/networking/mihomo/default.nix` | mihomo checks at startup whether another NIC already holds an IPv6 address and, finding none, skips the TUN's v6 half -- but `strict-route` still installs its v6 blackhole (`ip -6 rule add priority 9000 unreachable`). On a cold boot that rule lands before networkd's Router Solicitation is answered, so the host can never send RS/NS, never gets a GUA, and mihomo never re-checks: self-locking until the core restarts. The env var forces the TUN v6 address up, which installs mihomo's normal v6 rules whose `lookup main suppress_prefixlength 0` keeps on-link neighbor discovery and the RA default route off the TUN | Drop once mihomo either re-runs the IPv6 check or skips the v6 strict-route blackhole when the v6 TUN is disabled | 2026-10-10 | on a mihomo bump | active | +| WA-031 | `iommu=pt` kernel parameter — `hosts/idols-aquamarine/hardware-amd-gpu.nix` | ROCm's system requirements warn that multi-GPU systems may need IOMMU passthrough to prevent application hangs, and this host has two Radeon AI PRO R9700 cards. Added before the first inference run; not yet observed on gfx1201. | Drop once a ROCm release states that the default IOMMU mode is safe for multi-GPU gfx1201, or when the host runs a single GPU | 2026-10-13 | on a ROCm bump, or on the first hang report | active | +| WA-032 | `amdgpu.runpm=0` kernel parameter — `hosts/idols-aquamarine/hardware-amd-gpu.nix` | this board's DSDT advertises ATPX power control (`\_SB.PC00.GFX0.ATPX`) but never defines the `PEG0.PEGP._OFF`/`_ON` symbols its body calls, so the first idle resume of the headless R9700 leaves it in `runtime_status = error` (`resume of IP block failed -62`) until reboot, and amdgpu discards the ACPI error. Root cause, kernel-source trail and the measured field comparison: `hosts/idols-aquamarine/ROCM.md`. | Drop once a BIOS update defines those ACPI methods (upstream releases 1023/1066 exist for this board; a flash changes PCR7, so re-enroll TPM2), or a kernel release stops trusting ATPX when its power methods are missing | 2026-10-14 | on a BIOS update or kernel bump | active | If WA-008 freezes recur, the fallback is to restore the parameters commented out in `hosts/idols-ai/hardware-configuration.nix` — @@ -62,11 +64,13 @@ only delete inactive duplicates. A known limitation with no fix yet. -| ID | What & where | Why | Removal condition | Added | Revisit | Status | -| ------ | ------------------------------------------------------------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ----------------------------------------------------------------------------------------------------------------------------- | ---------- | -------------------------------------------------------------- | ------ | -| WA-016 | subscription rule snapshots — `modules/nixos/desktop/networking/mihomo/` (`rules_from:`) | mihomo's `proxy-providers` imports proxies only, so the `rules` and `proxy-groups` a subscription ships with are dropped by the core; the generator re-imports them from a local YAML copy that has to be refreshed by hand when the provider changes its routing policy | Drop the snapshots once mihomo can import rules from a proxy-provider, or the providers publish their rules as rule-providers | 2026-10-04 | when mihomo gains rule import | active | -| WA-022 | qemu-user binfmt removed — `hosts/12kingdoms-shoukei/hardware-configuration.nix` | the x86-on-arm module owns the x86 ELF handlers and this host does not need riscv64, but dropping `boot.binfmt.emulatedSystems` also drops `nix.settings.extra-platforms` and qemu-user, so `podman run --platform=linux/amd64`, x86 chroots and local builds of uncached x86 derivations need a remote x86_64 builder | Restore `boot.binfmt.emulatedSystems` (without x86_64/i686, which the module asserts against) if such a workload is needed | 2026-10-07 | when an amd64 container/chroot or uncached x86 build is needed | active | -| WA-025 | brcmfmac 4-way-handshake offload authentication — upstream kernel/wpa_supplicant, no code in this repo | Since hostap commit 416386060, wpa_supplicant 2.11+ expects `NL80211_CMD_PORT_AUTHORIZED` from drivers that offload the 4-way handshake; brcmfmac never reports it, so authentication intermittently fails on Apple Silicon (BRCM4378/4387/4388). nixpkgs already carries the Broadcom revert in wpa_supplicant 2.12; the proper fix, Janne Grunau's `wifi: brcmfmac: cfg80211: Report port_authorized for 4-way HS offload` (2026-09), is still under review. Entering the key again after such a failure only creates a duplicate profile, so cancel and retry instead | Fix lands via a nixpkgs kernel update (or by carrying the patch through `nixpkgs-patched`) | 2026-10-07 | on every kernel bump | active | +| ID | What & where | Why | Removal condition | Added | Revisit | Status | +| ------ | ---------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ----------------------------------------------------------------------------------------------------------------------------- | ---------- | -------------------------------------------------------------- | ------ | +| WA-016 | subscription rule snapshots — `modules/nixos/desktop/networking/mihomo/` (`rules_from:`) | mihomo's `proxy-providers` imports proxies only, so the `rules` and `proxy-groups` a subscription ships with are dropped by the core; the generator re-imports them from a local YAML copy that has to be refreshed by hand when the provider changes its routing policy | Drop the snapshots once mihomo can import rules from a proxy-provider, or the providers publish their rules as rule-providers | 2026-10-04 | when mihomo gains rule import | active | +| WA-022 | qemu-user binfmt removed — `hosts/12kingdoms-shoukei/hardware-configuration.nix` | the x86-on-arm module owns the x86 ELF handlers and this host does not need riscv64, but dropping `boot.binfmt.emulatedSystems` also drops `nix.settings.extra-platforms` and qemu-user, so `podman run --platform=linux/amd64`, x86 chroots and local builds of uncached x86 derivations need a remote x86_64 builder | Restore `boot.binfmt.emulatedSystems` (without x86_64/i686, which the module asserts against) if such a workload is needed | 2026-10-07 | when an amd64 container/chroot or uncached x86 build is needed | active | +| WA-025 | brcmfmac 4-way-handshake offload authentication — upstream kernel/wpa_supplicant, no code in this repo | Since hostap commit 416386060, wpa_supplicant 2.11+ expects `NL80211_CMD_PORT_AUTHORIZED` from drivers that offload the 4-way handshake; brcmfmac never reports it, so authentication intermittently fails on Apple Silicon (BRCM4378/4387/4388). nixpkgs already carries the Broadcom revert in wpa_supplicant 2.12; the proper fix, Janne Grunau's `wifi: brcmfmac: cfg80211: Report port_authorized for 4-way HS offload` (2026-09), is still under review. Entering the key again after such a failure only creates a duplicate profile, so cancel and retry instead | Fix lands via a nixpkgs kernel update (or by carrying the patch through `nixpkgs-patched`) | 2026-10-07 | on every kernel bump | active | +| WA-033 | USB-C/XHCI port `_PLD` ACPI noise — this board's DSDT, no code in this repo | every boot logs 16 `ACPI Error: Aborting method \_SB.PC00.TXHC.RHUB.SS01._PLD ... (AE_NOT_FOUND)` lines, plus `\_SB.PC02.XHCI.RHUB.HS01._PLD`; the ports work, and these messages are unrelated to the GPU's ATPX failure, which names `\_SB.PC00.GFX0.ATPX` instead | Drop if a BIOS update defines the `_PLD` descriptors | 2026-10-14 | on a BIOS update | active | +| WA-034 | IPv6-first resolution is dead on the wired link — `hosts/idols-aquamarine/default.nix` (`networking.getaddrinfo.precedence`) | the proxy gateway answers DNS in fake-IP mode and its IPv6 path is dead: an AAAA lookup returns a fake address whose connection fails at once (measured: `curl -6` fails in 8 ms, `curl -4` answers in 0.7 s). glibc clients are covered by the RFC 6724 table with the IPv4-mapped row raised; nix's libcurl and other non-glibc resolvers ignore `/etc/gai.conf`. Disabling IPv6 with `boot.kernel.sysctl` did not work here — networkd re-enables it on the wired link for `DHCP = "ipv6"`/`IPv6AcceptRA` — so that setting was dropped on 2026-10-14 | Drop once the gateway serves proxied names over IPv6 as well (the adjacent fix is `dns.ipv6: false` in mihomo, see WA-030) | 2026-10-14 | on a mihomo or gateway change | active | ## WIP diff --git a/hosts/12kingdoms-youko/homelab-services/README.md b/hosts/12kingdoms-youko/homelab-services/README.md index 9ef1041b..1030a80c 100644 --- a/hosts/12kingdoms-youko/homelab-services/README.md +++ b/hosts/12kingdoms-youko/homelab-services/README.md @@ -1,7 +1,7 @@ # Youko - Homelab Services Storage, operation and maintenance related services are running on this host (migrated here from the -retired `idols-aquamarine` guest): +retired `aquamarine` VM — the name now belongs to the inference server, not this host): 1. Storage such as a git server, file server, torrent downloader, etc. 1. Back up my personal data with restic; the data is encrypted on the client. diff --git a/hosts/12kingdoms-youko/homelab-services/default.nix b/hosts/12kingdoms-youko/homelab-services/default.nix index a7a924da..277a80ff 100644 --- a/hosts/12kingdoms-youko/homelab-services/default.nix +++ b/hosts/12kingdoms-youko/homelab-services/default.nix @@ -1,6 +1,6 @@ { mylib, ... }: { - # The homelab service modules, migrated here from the retired - # `hosts/idols-aquamarine` guest, now running directly on this host. + # The homelab service modules, migrated here from the retired `aquamarine` VM + # (the `hosts/idols-aquamarine` path is a physical host now). imports = mylib.scanPaths ./.; } diff --git a/hosts/12kingdoms-youko/homelab-services/monitoring/victoriametrics.nix b/hosts/12kingdoms-youko/homelab-services/monitoring/victoriametrics.nix index 533a6585..c96fcc8c 100644 --- a/hosts/12kingdoms-youko/homelab-services/monitoring/victoriametrics.nix +++ b/hosts/12kingdoms-youko/homelab-services/monitoring/victoriametrics.nix @@ -6,10 +6,7 @@ let # Powered-off hosts (SBCs) plus shoukei (exporter disabled on the # machine); remove entries when they come back online. - # `aquamarine` is decommissioned: its IP now lives on youko's bridge, so - # scraping it would relabel youko's metrics as aquamarine. offlineHosts = [ - "aquamarine" "shoukei" "suzu" "suzi" @@ -25,6 +22,7 @@ let # `vars/networking.nix`. physicalHosts = [ "ai" + "aquamarine" "shoryu" "shushou" "youko" diff --git a/hosts/README.md b/hosts/README.md index 2bdfea89..b54aea76 100644 --- a/hosts/README.md +++ b/hosts/README.md @@ -10,34 +10,40 @@ This directory contains all host-specific configurations for my NixOS and macOS Named after characters from "Oshi no Ko": -| Host | Platform | Hardware | Purpose | Status | -| ------- | --------------- | --------------------------------------- | ------------------ | --------- | -| `ai` | NixOS | Ultra 7 270K Plus + RTX 4090, 128G DDR5 | Gaming & Daily Use | ✅ Active | -| `kana` | NixOS (libvirt) | Virtual | Run AI Agents | ✅ Active | -| `ruby` | NixOS (libvirt) | Virtual | Run AI Agents | ✅ Active | -| `akane` | NixOS (aarch64) | Virtual (UTM) | aarch64 test VM | ✅ Active | +| Host | Platform | Hardware | Purpose | Status | +| ------------ | --------------- | ---------------------------------------------------- | ------------------ | --------- | +| `ai` | NixOS | i5-13600KF + RTX 4090, 32G DDR5 | Gaming & Daily Use | ✅ Active | +| `aquamarine` | NixOS | Ultra 7 270K Plus + 2x Radeon AI PRO R9700, 96G DDR5 | AI Inference | ✅ Active | +| `kana` | NixOS (libvirt) | Virtual | Run AI Agents | ✅ Active | +| `ruby` | NixOS (libvirt) | Virtual | Run AI Agents | ✅ Active | +| `akane` | NixOS (aarch64) | Virtual (UTM) | aarch64 test VM | ✅ Active | -On 2026-04-27 `ai` was rebuilt on a new platform. The MSI board and i5-13600KF it was added with in -2023-05 (the host was originally named `msi-rtx4090`) gave way to the Colorful CVN Z890 ARK FROZEN + -Intel Core Ultra 7 270K Plus. The traces in the history are PR #257 (the NIC moved from `enp5s0` to -`enp130s0`) and PR #258 (Niri output names renamed, `hardware-intel.nix` added). The kernel moved to -`linuxPackages_latest` in `77ca37ea`, the same day but outside #258. The 2×48G DDR5 kit was already -in place at that point; the repo has no record of when it went in. +On 2026-04-27 `ai` was rebuilt on a new platform. The MSI board it was added with in 2023-05, with +the i5-13600KF (the host was originally named `msi-rtx4090`), gave way to the Colorful CVN Z890 ARK +FROZEN + Intel Core Ultra 7 270K Plus. The traces in the history are PR #257 (the NIC moved from +`enp5s0` to `enp130s0`) and PR #258 (Niri output names renamed, `hardware-intel.nix` added). The +kernel moved to `linuxPackages_latest` in `77ca37ea`, the same day but outside #258. The 2×48G DDR5 +kit was already in place at that point; the repo has no record of when it went in. On 2026-10-06 `ai` grew from 2×48G (96G) to 2×48G + 2×16G (128G) across all four DIMM slots, at DDR5-4800 with XMP off — the only stable setting with four DIMMs on this board. The 16G pair must be -installed first; with the 48G pair in first the board does not POST. The bring-up order, the BIOS -settings, the stress-test recipe, and how to recover a board that will not POST are in -[`idols-ai/MIXED-MEMORY.md`](./idols-ai/MIXED-MEMORY.md). +installed first; with the 48G pair in first the board does not POST. It did run, in stretches of one +to two days at a time, before a sudden hard freeze, and resuming from suspend was unreliable too; +the stress test passed and no BIOS setting, DIMM or IMC fault was ever isolated, so 128G in four +DIMMs stayed untrusted. + +On 2026-10-13 that platform was split in two. The Z890 board, the 270K Plus, the 2×48G kit and two +Radeon AI PRO R9700 cards became `aquamarine`, a headless inference server; `ai` went back to the +MSI MAG B760M MORTAR board with the i5-13600KF, the 2×16G kit and the RTX 4090. `aquamarine` boots +from the NVMe that used to be `ai`'s `/data` disk. Its name and the address `192.168.5.101` are +reused from the `aquamarine` VM retired on 2026-09-21; that VM's services still run directly on +`youko` (`hosts/12kingdoms-youko/homelab-services/`). Hardware, install and runbook: +[`idols-aquamarine/README.md`](./idols-aquamarine/README.md). ##### Retired -| Host | Platform | Hardware | Purpose | Status | -| ------------ | --------------- | -------- | --------------------- | ---------- | -| `aquamarine` | NixOS (libvirt) | Virtual | Monitoring & Services | ⚪ Retired | - -`aquamarine` is retired; its services now run directly on `youko` -(`hosts/12kingdoms-youko/homelab-services/`). +_No physical host in this group is retired. The `aquamarine` VM (NixOS on libvirt, monitoring and +services) was retired on 2026-09-21 and its name was reused for the inference server above._ #### `darwin` - macOS Systems @@ -124,7 +130,7 @@ just build-host `eval-host` only evaluates the configuration. `build-host` builds the system closure without activating it. Host activation is always explicit: Colmena (`just shoryu`, `just shushou`, -`just youko`) from another machine, or `just local` on the machine itself. +`just youko`, `just aquamarine`) from another machine, or `just local` on the machine itself. ## Distributed Building diff --git a/hosts/idols-aquamarine/README.md b/hosts/idols-aquamarine/README.md new file mode 100644 index 00000000..53616bdf --- /dev/null +++ b/hosts/idols-aquamarine/README.md @@ -0,0 +1,235 @@ +# Host - Aquamarine + +Headless AI inference server. Revives the `aquamarine` name, retired on 2026-09-21 when the VM of +that name was decommissioned; this is a different, physical machine that reuses the name and the +`192.168.5.101` address. + +## Hardware + +| Part | Model | Notes | +| ------ | ----------------------------------------------------- | ------------------------------------------------------- | +| Board | Colorful CVN Z890 ARK FROZEN V20 (BIOS 1013) | moved from `ai` on 2026-10-13, so its EFI vars come too | +| CPU | Intel Core Ultra 7 270K Plus | Arrow Lake; NPU unused here | +| Memory | 2x48G JUHOR (96G), 4800 MT/s | the 2x16G kit stayed with `ai` | +| GPU | 2x AMD Radeon AI PRO R9700, 32G each (RDNA4, gfx1201) | 64G of VRAM total | +| Root | Fanxiang S790 2TB | was `ai`'s `/data` disk until 2026-10-13 | +| NIC | Realtek RTL8125BG 2.5GbE, `enp133s0` today | matched by the `en*` prefix | + +Two things about this board set the ceiling for the GPU work: + +- The second full-length x16 slot is chipset-attached: its root port negotiates Gen4 x4, while the + card side reports Gen5 x16 either way (./ROCM.md has the link measurements). The chipset card + probes **7.1 GB/s** host->device against 46.3 GB/s for the CPU-attached card, so a two-card layer + split is 8-27% slower per request and ~9% ahead on six concurrent requests. Read the inference + engine's `PCIe probe` line rather than `lspci`, and prefer a CPU-attached slot for a second card. +- Multi-GPU work should split by layer (RPC / pipeline parallel), not by tensor parallelism: the + R9700 has no xGMI link between cards, and a dual-R9700 report has RCCL deadlocking with vLLM TP=2 + even with `NCCL_P2P_DISABLE=1`. See [ROCM.md](./ROCM.md). + +The iGPU needs no configuration: the kernel binds `i915` and +`hardware.enableRedistributableFirmware` is on, so a monitor plugged into the board's HDMI/DP ports +gets the text console (`multi-user.target`, no display manager) when SSH is unreachable. + +`iommu=pt` is set in [hardware-amd-gpu.nix](./hardware-amd-gpu.nix) because ROCm documents hangs on +multi-GPU systems without IOMMU passthrough. The same file disables amdgpu runtime power management: +this board's DSDT advertises ATPX power control that its own ACPI methods cannot deliver, so the +first idle resume of the headless R9700 kills it until reboot. [ROCM.md](./ROCM.md) records whose +defect that is, the supported ROCm versions, and the inference notes. + +## Disk layout + +Declarative via [disko-fs.nix](./disko-fs.nix): ESP + LUKS (`aqua-luks`) + btrfs subvolumes (`@nix`, +`@persistent`, `@snapshots`, `@tmp`, `@swap` with a 96G swapfile), ephemeral tmpfs root, state under +`/persistent` via [preservation.nix](./preservation.nix). Secure Boot via lanzaboote +([secureboot.nix](./secureboot.nix)). + +The mapper is `aqua-luks`, not `nixos-luks`: the disk is formatted while the staging host is +running, and that host already owns the `nixos-luks` name. + +## Install (staged from a running host) + +This disk is installed into a machine that is not the one it gets formatted on: `disko` runs while +the disk is still attached to a running NixOS host (`ai` below), and the disk then moves to its own +machine. The target machine is the board the install runs on, so the EFI boot entry and the Secure +Boot key enrollment the host already has carry over. That is what makes a staged install possible; +it also means these steps run on a machine whose own root disk is one `disko destroy` away. + +Steps 1 to 6 only touch this disk and leave `ai` running. `ai`'s own deploy and shutdown are step 7, +and they belong together: the machine comes back on the new board with this disk gone. + +1. Confirm the target disk and release it: `lsblk -f` must show no mount and no open mapper on it. A + disk reused from another role still carries that role's mounts: + + ```bash + ls -l /dev/disk/by-id/nvme-Fanxiang_S790_2TB_FXS790254050582 + lsblk -f + sudo umount /data # the previous role's mountpoint + sudo cryptsetup close data-luks # and its mapper + ``` + +2. Partition, encrypt and mount. This **destroys everything on that disk** — re-read the `device` + line in [disko-fs.nix](./disko-fs.nix) first. Run it as **root**: unprivileged, disko still walks + through the confirmation prompt and then fails on every step with `Permission denied`. + + ```bash + cd ~/nix-config + sudo nix run github:nix-community/disko -- --mode destroy,format,mount hosts/idols-aquamarine/disko-fs.nix + ``` + + Success looks like two new partitions — `disk-nixos-aquamarine-ESP` (vfat, 600M) and + `disk-nixos-aquamarine-root` (crypto_LUKS) — and seven mounts under `/mnt`. Check before moving + on: + + ```bash + lsblk -o NAME,SIZE,FSTYPE,PARTLABEL /dev/nvme0n1 + findmnt -R /mnt + ``` + +3. Reuse the staging host's Secure Boot PKI bundle, so lanzaboote signs this system with a key the + firmware already trusts. The bundle contains a private signing key: copy it yourself, and never + commit it or send it through chat. + + ```bash + sudo mkdir -p /mnt/persistent/etc/secureboot + sudo cp -a /persistent/etc/secureboot/. /mnt/persistent/etc/secureboot/ + + # lanzaboote reads the key from /etc/secureboot inside the target, and preservation + # only bind-mounts that directory at boot — so expose it during the install too. + sudo mkdir -p /mnt/etc/secureboot + sudo mount --bind /mnt/persistent/etc/secureboot /mnt/etc/secureboot + ``` + + `nixos-install` installs the boot loader through `nixos-enter --root /mnt` and + `NIXOS_INSTALL_BOOTLOADER=1 ... switch-to-configuration boot`, so lanzaboote's install hook reads + `/etc/secureboot/keys/db/db.key` **from the target root**. Without the bind mount that path is an + empty directory and the install fails at "installing the boot loader", however good the copy in + `/mnt/persistent` is. + + Skipping this is fine only if you would rather start over: delete the firmware keys, + `sbctl create-keys` + `sbctl enroll-keys` in setup mode. The fresh keys must end up in + `/etc/secureboot` — lanzaboote's `pkiBundle`, while `sbctl create-keys` writes to its own keydir + — or the firmware trusts a key this system's UKI was not signed with. That changes PCR7, so any + TPM2 LUKS enrollment made before it stops unlocking. + +4. Enroll the new LUKS volume with this board's TPM2, keeping the passphrase slot as the fallback: + + ```bash + sudo systemd-cryptenroll --tpm2-device=auto --tpm2-pcrs=0+7 \ + /dev/disk/by-id/nvme-Fanxiang_S790_2TB_FXS790254050582-part2 + sudo systemd-cryptenroll /dev/disk/by-id/nvme-Fanxiang_S790_2TB_FXS790254050582-part2 # slots: expect tpm2 + passphrase + ``` + + PCR7 measures the Secure Boot key database, so enroll **after** step 3, not before. Re-run this + whenever the Secure Boot keys change. + +5. Install the real system. The closure is mostly cached on the staging host, and step 3's bind + mount is what lets lanzaboote sign into this disk's ESP: + + ```bash + cd ~/nix-config + sudo nixos-install --root /mnt --flake .#aquamarine --no-root-password + ``` + + If a private flake input is not in the host's store yet, fetch it as root: `sudo` does not pass + `SSH_AUTH_SOCK` through on this host, so run the install from a root shell with its own agent + (`sudo su`, then `eval "$(ssh-agent)" && ssh-add ~/.ssh/`). + + The installer writes an EFI boot entry into **this board's NVRAM** and may put it first. Do not + reboot the staging host with both disks attached — it would boot aquamarine's system. + + The install generates the target's `/etc`, but preservation only bind-mounts `/persistent` at + boot, so the state it looks for has to be in place before the disk is unmounted. Move the + generated `/etc/ssh` and `machine-id` in, then read the listing: + + ```bash + sudo nixos-enter --root /mnt + mkdir -p /persistent/etc + mv /etc/ssh /persistent/etc/ + mv /etc/machine-id /persistent/etc/ + ls -l /persistent/etc/ssh /persistent/etc/ssh/authorized_keys.d + exit + ``` + + `authorized_keys.d/` must be a **regular file** with mode `0444`: sshd's `StrictModes` + rejects a key file whose path crosses `/nix/store`, which is group-writable (`1775`), so a + symlink to `/etc/static/...` there locks every login out. `sshd_config` is a symlink to + `/etc/static/ssh/sshd_config` on purpose — that one tracks the current generation. If a file is + missing, copy the real one in and create its directory if needed: + `cp -L /etc/static/ssh/authorized_keys.d/. /persistent/etc/ssh/authorized_keys.d/`. + +6. Unmount the target: `umount -R /mnt` also drops step 3's bind mount. This disk is done; leave the + staging host running for step 7. + + ```bash + sudo umount -R /mnt + findmnt -R /mnt # nothing left mounted + ``` + +7. On `ai`, stage its next generation **and** shut it down — one step, never one without the other. + The machine boots on the new board with this disk gone, so the generation it comes back on must + not wait for a LUKS volume here; and staging only takes effect at that boot, so shutting down + first would leave the current generation as the default. + + ```bash + # the host's initrd must no longer list this disk's mapper + nix eval --json .#nixosConfigurations.ai-niri.config.boot.initrd.luks.devices --apply builtins.attrNames + # [ "nixos-luks" ] + + just niri boot # stage only: `switch` would apply the NIC rename now and cut the LAN + sudo poweroff # only after the deploy succeeded + ``` + + Then move the disk to its machine, in the same M.2 slot: the EFI entry names the disk by its + device path, and a different slot can invalidate it. + +## First boot on aquamarine + +1. The initrd should unlock `aqua-luks` from the TPM; if it prompts, use the passphrase. +2. If the firmware has no boot entry for the disk, pick it once from the boot menu (the ESP also + carries the `\EFI\BOOT\BOOTX64.EFI` fallback), then let the first activation write the entry. +3. From another host: + + ```bash + ssh aquamarine ip addr + ssh aquamarine systemctl --failed + ssh aquamarine journalctl -b -p err + ssh aquamarine rocminfo + ssh aquamarine rocm-smi --showproductname + ``` + + If `ssh` refuses the connection, or `sshd` logs `bad ownership or mode for directory /nix/store`, + the `/etc/ssh` copy under `/persistent` is incomplete — repeat step 5's check from the console + before deploying anything else. + + `ip addr` must show `192.168.5.101` on the wired interface. The networkd match uses the `en*` + prefix, so a renamed NIC cannot strand the host; if the address is missing, check the cable and + `ip -br link` first. + +## After the first boot + +The install created a fresh SSH host key; the old `aquamarine` VM key is dead and the new one is +already in `vars/networking.nix` under `knownHosts` (2026-10-13). Redo these steps only after a +reinstall or a key rotation: + +1. `ssh aquamarine cat /etc/ssh/ssh_host_ed25519_key.pub`. +2. Put that key in `vars/networking.nix` under `knownHosts` (the entry was removed with the old key) + and, if this host should read agenix secrets, in `~/codes/nix-secrets/secrets.nix` under + `application_server_keys`. +3. Rekey and bump the pin: `just -f secrets/Justfile rekey`, push `nix-secrets`, then + `just upp mysecrets` here. + +This config enables no `modules.secrets.*` group, so aquamarine decrypts nothing today; adding a +group is what requires the recipient step above. + +## Deploy + +```bash +just aquamarine # colmena, from any other host +just local # on aquamarine itself +``` + +## Rollback + +The install only touches the target disk; the staging host keeps booting from its own. To undo, move +the disk back, restore the `hosts/idols-ai/disko-fs-data.nix` module this change deleted, and +re-enroll that machine's TPM2 slot — see `../idols-ai/README.md`. diff --git a/hosts/idols-aquamarine/ROCM.md b/hosts/idols-aquamarine/ROCM.md new file mode 100644 index 00000000..e4219d2b --- /dev/null +++ b/hosts/idols-aquamarine/ROCM.md @@ -0,0 +1,273 @@ +# ROCm on aquamarine + +`aquamarine` runs Radeon AI PRO R9700 cards (Navi 48, RDNA4, PCI ID `1002:7551`, ROCm target +`gfx1201`) as headless compute devices; the Intel iGPU drives the console. This note records which +versions support the card, why amdgpu runtime power management leaves it unusable, and which +decisions that produced. Claims are marked **verified** (primary source), **community-reported**, or +**inference** (read from kernel source, not confirmed by a maintainer). + +## Versions + +- **ROCm**: AMD's compatibility matrix has no RDNA4 content and no `gfx1201` in the 6.4.0 column; + `gfx1201` and the RDNA4 row appear from 7.0.0 on. The R9700 first appears in AMD's _Radeon_ matrix + for **6.4.4** (Ubuntu 24.04.2/22.04.5, RHEL 9.6/9.4), so it arrived inside 6.4.x, not at 6.4.0. + **verified** + ([matrix](https://github.com/ROCm/legacy-rocm-build/blob/develop/docs/compatibility/compatibility-matrix.rst), + [6.4.4 Radeon matrix](https://rocm.docs.amd.com/projects/radeon-ryzen/en/docs-6.4.4/docs/compatibility/compatibilityrad/native_linux/native_linux_compatibility.html)) +- **ROCm today**: system requirements list `AMD Radeon AI PRO R9700 | RDNA4 | gfx1201 | ✅`. + **verified** + ([requirements](https://rocm.docs.amd.com/projects/install-on-linux/en/latest/reference/system-requirements.html)) +- **This repository**: nixpkgs `f45c6f0` provides `rocmPackages.clr` and `rocmPackages.rocm-smi` + 7.2.3 and Mesa 26.2.4; the host runs the nixpkgs default kernel (6.18.x LTS), not + `linuxPackages_latest` (7.2.8), which is what the first bring-up booted. **verified** (checked by + `nix eval`) +- **Mesa**: "initial GFX12 (RDNA4) support on RADV" landed in 25.0.0. **verified** + ([notes](https://gitlab.freedesktop.org/mesa/mesa/-/blob/main/docs/relnotes/25.0.0.rst)) +- **Kernel**: amdgpu has no static `1002:7551` entry; its ID table ends with + `PCI_DEVICE(0x1002, PCI_ANY_ID)` wildcards carrying `CHIP_IP_DISCOVERY`, so binding depends on the + `soc24`/`gfx_v12_0` IP blocks and firmware, not a PCI ID. AMD validates Ubuntu 24.04.3 HWE kernel + 6.14 for R9700 with ROCm 6.4.4; no primary source states a minimum kernel for Navi 48. + **verified** + ([amdgpu_drv.c](https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git/tree/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c?h=v7.2.8)) +- **linux-firmware**: Navi 4x uses `gc_12_0_1_*`, `psp_14_0_3_*`, `smu_14_0_3.bin`, + `sdma_7_0_1.bin`, `dcn_4_0_1_dmcub.bin`; an Ubuntu SRU "Fix power state transition on navi4x" + updates exactly those files and states they fix bad D3cold→D0 transitions. **verified** + ([LP #2122659](https://bugs.launchpad.net/ubuntu/+source/linux-firmware/+bug/2122659)) + +## The runtime-PM failure + +The chain is readable from kernel source: a defect in this board's ACPI firmware that the driver +walks into and then hides. The R9700 and the ROCm userspace are fine. + +The defect is in Colorful's DSDT. It advertises the ATPX dGPU power-control method while the method +body calls `\_SB.PC00.PEG0.PEGP._OFF`/`_ON`, symbols this DSDT never defines, so the firmware +promises power control it cannot deliver. amdgpu then hides the failure instead of surfacing it: it +routes runtime PM through ATPX and discards the ACPI error (step 4), which turns "this firmware +cannot power-gate the card" into "the card is dead until reboot". + +Hitting it took exactly two display-class devices (the iGPU kept for the console fallback plus one +R9700) for `amdgpu_atpx_detect()` to accept ATPX, this DSDT, and an idle dGPU with no display +attached. With **two** R9700s installed there are three display-class devices, so that condition no +longer holds and `amdgpu.runpm=0` may be belt-and-braces rather than a live fix; it stays because +disabling runtime PM on an always-on host costs nothing and the failure needs a reboot to clear. The +field comparison below predates the second card. + +1. `amdgpu_atpx_detect()` accepts an ACPI `ATPX` method only when two VGA/display-class devices + exist, which the iGPU plus one R9700 satisfies; this DSDT does expose `\_SB.PC00.GFX0.ATPX`, as + the log names it. **verified** + ([atpx handler](https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git/tree/drivers/gpu/drm/amd/amdgpu/amdgpu_atpx_handler.c?h=v7.2.8)) +2. When ATPX advertises dGPU power control, amdgpu sets `AMD_IS_PX` on the R9700 and, with the + default `runpm=-1`, selects `AMDGPU_RUNPM_PX` ("Using ATPX for runtime pm"). **verified** + ([amdgpu_device.c](https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git/tree/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c?h=v7.2.8)) +3. `vga_switcheroo_init_domain_pm_ops()` routes runtime suspend/resume through the ATPX handler, + which evaluates `\_SB.PC00.PEG0.PEGP._OFF`/`_ON`. Those symbols are absent here, so ACPI aborts + the method with `AE_NOT_FOUND`. **verified** + ([vga_switcheroo.c](https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git/tree/drivers/gpu/vga/vga_switcheroo.c?h=v7.2.8)) +4. The failure is discarded twice over: `amdgpu_atpx_power_state()` calls + `amdgpu_atpx_set_discrete_state()` and then returns 0 unconditionally, and + `vga_switcheroo_power_switch()` never looks at the handler's return value either. So the failed + switch is silent; the driver resumes as if the card had been power-cycled, re-initializes SMU, + gets no answer (`SMU: No response msg_reg: 6 resp_reg: 0`) and reports + `resume of IP block failed -62` with `runtime_status = error`. **verified** for the two + discarded returns, **inference** for the SMU sequence that follows + ([amdgpu_atpx_handler.c](https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git/tree/drivers/gpu/drm/amd/amdgpu/amdgpu_atpx_handler.c?h=v6.18.54), + [vga_switcheroo.c](https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git/tree/drivers/gpu/vga/vga_switcheroo.c?h=v6.18.54)). +5. With `runtime_status` not `active`, rocm-smi prints its low-power-state warning; + `check_runtime_status()` reads `…/device/power/runtime_status`. **verified** + ([rocm_smi.py](https://github.com/ROCm/rocm-systems/blob/rocm-7.2.3/projects/rocm-smi-lib/python_smi_tools/rocm_smi.py)) + +**No upstream report of this exact chain was found.** Searches of the freedesktop drm/amd tracker, +kernel Bugzilla and distro trackers found no bug matching `PEGP._OFF` + `AE_NOT_FOUND` + +`Aborting method … ATPX` on RDNA4 desktop hardware. The nearest documents are the amd-gfx patch +"drm/amdgpu/soc24: reset dGPU if suspend got aborted", which handles the same `SMU … -62` tail but +only when `adev->in_s3` is set (present in 7.2.8; it does **not** cover runtime PM), and the navi4x +firmware SRU. **verified** +([patch](https://lists.freedesktop.org/archives/amd-gfx/2026-June/146878.html), +[soc24.c](https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git/tree/drivers/gpu/drm/amd/amdgpu/soc24.c?h=v7.2.8)) + +Nor is a fix in flight on the code itself: `amdgpu_atpx_handler.c`'s most recent change is a 2026-05 +buffer-length hardening, and its dGPU power-control semantics have not been touched since 2019, +while the 2025-2026 `vga_switcheroo.c` changes are cosmetic (`pci_is_display()`, fbcon wrappers, +`kmalloc_obj` conversions). **verified** (mainline commit history of both files) + +## What actually fixes it + +- `amdgpu.runpm=0` is the documented "disable" value (default `-1`, auto). With mode `NONE`, + `amdgpu_pmops_runtime_checks()` calls `pm_runtime_forbid()` and returns `-EBUSY`, the + vga_switcheroo domain is not installed, and no runtime suspend can run. **verified** + ([amdgpu_drv.c](https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git/tree/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c?h=v7.2.8)) +- `echo on > /sys/bus/pci/devices/…/power/control` is the PM-core equivalent — the device never + runtime-suspends — but it leaves the driver registered as a PX client, which is why this host sets + `amdgpu.runpm=0` instead (below) rather than persisting that write with a udev rule. **verified** + for the ABI, **community-reported** as a fix. +- AMD's own notes carry the workaround ("users can run amdgpu.runpm=0, which temporarily disables + the runtime power management feature from the driver"), and the Arch Wiki gives the same advice + for a dGPU that "becomes disabled during use", citing drm/amd issue 1820. **verified** + ([ROCm 5.0.0 highlights](https://github.com/ROCm/legacy-rocm-build/blob/develop/tools/autotag/templates/highlights/5.0.0.md), + [Arch Wiki](https://wiki.archlinux.org/title/AMDGPU#Issues_with_power_management_/_dynamic_re-activation_of_a_discrete_amdgpu_graphics_card)) + +The workaround is field-verified on this host: the pre-fix boot logged +`amdgpu: Using ATPX for runtime pm` and left the card in `runtime_status = error`, while the 6.18.54 +boot with `amdgpu.runpm=0` logs `amdgpu: runtime pm is manually disabled` and +`amdgpu: Runtime PM not available`, keeps `power/control = on` and `runtime_status = active`, and +`rocm-smi` reads the card normally after minutes of idling. **verified** (`journalctl -k -b 0` vs +`-b -2` on this host) + +One adjacent problem gets conflated with this one: Navi 48 s2idle breaks when +`hardware.amdgpu.overdrive`'s default `ppfeaturemask = "0xfffd7fff"` disables GFXOFF (upstream +default `0xfff7bfff`; `0xfff7ffff` works). **verified** +([nixpkgs#564601](https://github.com/NixOS/nixpkgs/issues/564601)) + +`amdgpu.ppfeaturemask=0xffffffff` does not address the ATPX/SMU path; it comes from a community +tuning repository and no primary source ties it to it. **community-reported** +([r9700-setup](https://github.com/stew675/r9700-setup/blob/HEAD/README.md)). The host's own +`pcie_aspm=off` and `processor.max_cstate=2` are unrelated to this path too — they are set for PCIe +link retraining and C-state exit latency, with their reasons in +[hardware-amd-gpu.nix](./hardware-amd-gpu.nix). + +## Kernel parameters and module options + +- `iommu=pt` — keep. AMD: "Systems with multiple GPUs may require iommu=pt … to prevent application + hangs". **verified** + ([requirements](https://rocm.docs.amd.com/projects/install-on-linux/en/latest/reference/system-requirements.html)) +- `iommu=off` — claimed by `stew675/r9700-setup` to "allow for P2P between multiple R9700's"; no AMD + statement supports that. **community-reported** +- `amdgpu.gpu_recovery=1` — enables GPU recovery; default is `-1`, "auto, disabled except SRIOV". + **verified** + ([amdgpu_drv.c](https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git/tree/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c?h=v7.2.8)) +- `amdgpu.ppfeaturemask` — exposed as `hardware.amdgpu.overdrive.ppfeaturemask`; see the s2idle + caveat above. **verified** + ([module](https://github.com/NixOS/nixpkgs/blob/master/nixos/modules/services/hardware/amdgpu.nix)) +- `amdgpu.ras_enable` — the real parameter name (`module_param_named(ras_enable, …)`), default `-1`; + the driver's GECC message calls it `amdgpu_ras_enable`. **verified** + ([amdgpu_psp.c](https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git/tree/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c?h=v7.2.8)) + +The Arch Wiki documents the same knobs plus `power_dpm_force_performance_level` and the COMPUTE +profile, but has no R9700/gfx1201 entry; its ROCm section only says the official list is short and +equivalent-generation consumer cards are "known to work", and never mentions +`HSA_OVERRIDE_GFX_VERSION`. **verified** ([AMDGPU](https://wiki.archlinux.org/title/AMDGPU), +[GPGPU § ROCm](https://wiki.archlinux.org/title/General-purpose_computing_on_graphics_processing_units#ROCm)) + +## ROCm userland in nixpkgs + +- `hardware.amdgpu.opencl.enable = true` adds both `rocmPackages.clr` and `rocmPackages.clr.icd` — + the module's own shape — rather than `.icd` alone. **verified** + ([amdgpu.nix](https://github.com/NixOS/nixpkgs/blob/master/nixos/modules/services/hardware/amdgpu.nix)) +- **`rocm-smi` works on a healthy card.** The `Fail to open libdrm_amdgpu.so.1` message upstream + logs (and AMD's amdsmi PR tolerates, closed unmerged) appeared here only while the R9700 sat in + `runtime_status = error`, so it is a symptom of the wedged card rather than a packaging bug. With + `amdgpu.runpm=0` the tool reads temperatures, clocks, power and VRAM, and `--showproductname` + answers `Card Model: 0x7551`, `Card Vendor: 0x1002`, `GFX Version: gfx1201`; only `Card Series` + stays `N/A` because the userspace name table has no R9700 entry. **verified** (observed on this + host) ([amdsmi PR](https://github.com/ROCm/rocm-systems/pull/2987)) +- nixpkgs' Ollama and Open WebUI modules use `SupplementaryGroups = [ "render" ]`, matching this + host. **verified** + ([ollama.nix](https://github.com/NixOS/nixpkgs/blob/master/nixos/modules/services/misc/ollama.nix)) +- A real NixOS RDNA4 host pins `linuxPackages_latest` and sets `amdgpu.runpm=0` ("Disable runtime + power management for stability") plus `amdgpu.gpu_recovery=1` and `amdgpu.dc=1`. + **community-reported** + ([config](https://github.com/dustinlyons/nixos-config/blob/main/hosts/nixos/default.nix)) +- GitHub code search found **no** public Nix configuration targeting `gfx1201` or the R9700. + +## Two R9700s + +- R9700 cards have no Infinity Fabric/xGMI link. A dual-R9700 report documents an **RCCL deadlock + with vLLM TP=2** on ROCm 7.2.1 (RCCL 2.27.7): both GPUs spin at 100%, `NCCL_P2P_DISABLE=1` changes + nothing, and the same stack worked with RCCL 2.27.3. **verified** (open upstream) + ([rocm-systems#5480](https://github.com/ROCm/rocm-systems/issues/5480)) +- llama.cpp gates peer copies behind `GGML_CUDA_NO_PEER_COPY`; without it cross-device copies use + `cudaMemcpyPeerAsync` and the backend advertises P2P events. The tuning repo recommends + `-DGGML_CUDA_NO_PEER_COPY=1` "if P2P is unstable in your topology". **verified** (flag), + **community-reported** (recommendation) + ([ggml-cuda.cu](https://github.com/ggml-org/llama.cpp/blob/master/ggml/src/ggml-cuda/ggml-cuda.cu)) +- `HIP_VISIBLE_DEVICES` (documented by llama.cpp) and `ROCR_VISIBLE_DEVICES` (used as + `ROCR_VISIBLE_DEVICES=0` in the R9700 report) both hide the iGPU. **verified** + ([build docs](https://github.com/ggml-org/llama.cpp/blob/master/docs/build.md), + [legacy-rocm-build#6630](https://github.com/ROCm/legacy-rocm-build/issues/6630)) +- Split by layer/pipeline, not tensor parallel: `-sm/--split-mode` offers `none`, `layer` (default, + "split layers and KV across GPUs (pipelined)"), `row` and `tensor` (parallelised, EXPERIMENTAL); + `--device` and `--main-gpu` choose devices. **verified** + ([arg.cpp](https://github.com/ggml-org/llama.cpp/blob/master/common/arg.cpp)) +- Both installed R9700s (`0000:04:00.0` behind the card's own switch at `02:00.0`/`03:00.0`, and + `0000:83:00.0` behind `82:00.0`) negotiate `current_link_width = 16` at `32.0 GT/s` (PCIe 5.0 x16, + their maximum) — that is the card-side link, not the slot. The slot links differ: the CPU root + port `0000:00:06.0` is x16 at 32 GT/s, while the chipset root port `0000:80:1c.0` is **x4 at 16 + GT/s (PCIe 4.0)**. The board manual's x4 claim is therefore measured, and because `lspci` on the + card reports x16 either way, read the root port or the engine's `PCIe probe` line (./README.md). + +## Inference software + +- **llama.cpp**: `-DGGML_HIP=ON -DGPU_TARGETS=gfx1201` (`GPU_TARGETS` is optional; omitting it + builds for all GPUs present). `HSA_OVERRIDE_GFX_VERSION` is documented only for GPUs that are not + officially supported, so it is not needed. ROCm's matrix validates llama.cpp `b6356` with 7.0.x. + **verified** ([build docs](https://github.com/ggml-org/llama.cpp/blob/master/docs/build.md), + [matrix](https://github.com/ROCm/legacy-rocm-build/blob/develop/docs/compatibility/compatibility-matrix.rst)) +- **llama.cpp on gfx1201 is not clean**: an upstream report on one R9700 shows Vulkan passing while + ROCm 7.14 hangs in generation and later resets MES/PSP (`resume of IP block failed -62`), + still open. **verified** ([#6630](https://github.com/ROCm/legacy-rocm-build/issues/6630)) +- The tuning repo also passes `-DGGML_HIP_ROCWMMA_FATTN=ON` and `-DGGML_HIP_RCCL=ON`; only + `GGML_HIP_RCCL` exists in `ggml/CMakeLists.txt`, so the ROCWMMA flash-attention flag is stale and + CMake ignores it. **community-reported** + ([CMakeLists.txt](https://github.com/ggml-org/llama.cpp/blob/master/ggml/CMakeLists.txt)) +- **Ollama**: the PR "Add native support for AMD RDNA 4 (gfx1201 / Radeon AI Pro R9700)" whitelists + the architecture and adds ROCm targets but is **closed and not merged**; it states RDNA4 discovery + timed out in the HIP handshake before whitelisting and links reports of + `TensileLibrary_lazy_gfx1201.dat` missing. A later report shows Ollama 0.20.2-rocm detecting + `compute=gfx1201` while still logging the rocBLASLt `.dat` error. **verified** + ([#14021](https://github.com/ollama/ollama/pull/14021), + [#15338](https://github.com/ollama/ollama/issues/15338)) +- **vLLM**: the current ROCm GPU installation page does not mention gfx1201, RDNA4, R9700 or 9070; + working dual-R9700 setups are community forks. **verified** (docs' silence) + ([vLLM docs](https://docs.vllm.ai/en/latest/getting_started/installation/gpu.html)); + **community-reported** ([toolboxes](https://github.com/kyuz0/amd-r9700-vllm-toolboxes)) + +## Power, thermals, ECC + +- **GECC** is a PSP boot-config bit: the driver prints `GECC is enabled`, or + `GECC will be enabled in next boot cycle` after setting it; enabling needs UMC RAS support and + `ras_default_ecc_enabled`/`amdgpu.ras_enable=1`. It takes effect next boot and costs VRAM + capacity/bandwidth, so decide before filling the cards. **verified** + ([amdgpu_psp.c](https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git/tree/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c?h=v7.2.8)) +- **Monitoring/control**: `power_dpm_force_performance_level`, `pp_dpm_*` clocks, + `gpu_busy_percent`, `mem_busy_percent` and the `fan_*` files are documented. **verified** + ([kernel thermal docs](https://docs.kernel.org/gpu/amdgpu/thermal.html)); the Arch Wiki adds + `pp_od_clk_voltage`, `pp_power_profile_mode` (COMPUTE is profile 5) and warns that PowerPlay-table + tools are "bugged on kernel side" for Navi 3x/4x. **verified** + ([Arch Wiki](https://wiki.archlinux.org/title/AMDGPU#Overclocking)) +- **Undervolt/power cap**: the tuning repo forces `manual`, writes `vo -85` + `c` to + `pp_od_clk_voltage`, and writes watts to hwmon `power1_cap` from a systemd oneshot. No NixOS + module exists. **community-reported** + ([tune_r9700.sh](https://github.com/stew675/r9700-setup/blob/HEAD/tune_r9700.sh)) + +## What this repository does, and what is still open + +1. `hardware-amd-gpu.nix` sets `"amdgpu.runpm=0"` next to `iommu=pt`, over a `power/control=on` udev + rule, which would leave the driver registered as a PX client. Revisit if a BIOS update defines + the `_OFF`/`_ON` methods — or now that two cards may keep ATPX out of the picture (above). +2. `iommu=pt` stays; `iommu=off` for P2P is a community claim with no AMD statement. +3. `hardware.amdgpu.opencl.enable = true` is the shape the host uses — the module adds both `clr` + and `clr.icd` — instead of a hand-written `hardware.graphics.extraPackages`. +4. `rocmPackages.rocm-smi` is not wrapped for libdrm. It reads a healthy card completely; the + `Fail to open libdrm_amdgpu.so` message only appeared while the card was wedged (see "ROCm + userland in nixpkgs"). Expect `Card Series: N/A` until upstream names the R9700. +5. `hardware.amdgpu.overdrive` stays off unless undervolting is wanted; if it is turned on, set + `ppfeaturemask = "0xfff7ffff"` (nixpkgs#564601). Power-cap scripting stays out of Nix. + `amdgpu.gpu_recovery=1` is set, because the driver default is auto/off except SRIOV. +6. **Open:** set `HIP_VISIBLE_DEVICES` (and `ROCR_VISIBLE_DEVICES` where honoured) so the iGPU is + never selected; for llama.cpp prefer `--split-mode layer` or an RPC/pipeline split over tensor + parallelism, and build with `-DGGML_HIP=ON -DGPU_TARGETS=gfx1201`, no `HSA_OVERRIDE_GFX_VERSION`. +7. Keep the x4 claim in [README.md](./README.md) and `hardware-amd-gpu.nix`: the chipset root port + `80:1c.0` measures 16 GT/s x4 while both cards' own links are x16, which is why the engine's + probe matters. Keep "split by layer, not tensor parallel" either way — it now rests on the RCCL + TP=2 issue, not slot width. +8. **Open:** make the ECC decision explicit: for GECC add `amdgpu.ras_enable=1` and confirm the boot + log's `GECC is enabled` line; otherwise say so, because the kernel warns about it every boot. + +## Not settled from primary sources + +- The `PEG0.PEGP._OFF` → ATPX abort → SMU `-62` sequence is read from kernel source; no maintainer + has confirmed the "ignored ATPX return value" step, and no upstream report of the chain was found. +- No primary source states a minimum kernel for Navi 48, and none states that PCIe P2P between R9700 + cards is available — only the two-card report and llama.cpp's opt-out flag. +- vLLM's official ROCm support for gfx1201 is undocumented; working configurations are community + forks whose correctness is not independently verified. diff --git a/hosts/idols-aquamarine/default.nix b/hosts/idols-aquamarine/default.nix new file mode 100644 index 00000000..260c1c3f --- /dev/null +++ b/hosts/idols-aquamarine/default.nix @@ -0,0 +1,139 @@ +{ + disko, + myvars, + lib, + ... +}: +############################################################# +# +# Aquamarine - headless AI inference server. Hardware, install and runbook: +# ./README.md. +# +############################################################# +let + hostName = "aquamarine"; # Define your hostname. + + inherit (myvars.networking) proxyGateway proxyGateway6 nameservers; + inherit (myvars.networking.hostsAddr.${hostName}) ipv4; + ipv4WithMask = "${ipv4}/24"; +in +{ + imports = [ + disko.nixosModules.default + # disks + ./disko-fs.nix + + # hardware + ./hardware-configuration.nix + ./hardware-amd-gpu.nix + + ./preservation.nix + ./secureboot.nix + ]; + + # Zram consumes physical memory for compression, which deadlocks when the + # model being loaded approaches the physical memory limit — exactly the + # workload this host exists for. Use the disk swapfile that disko creates at + # /swap instead, and keep zswap as a compressed cache in front of it so cold + # anonymous pages stay in RAM rather than hitting the SSD. + modules.zram.enable = false; + boot.zswap = { + enable = true; + compressor = "zstd"; + # 10% of 96G; the module default 25% would compete with the page cache that + # holds the model weights being read in. + maxPoolPercent = 10; + }; + + # Local btrfs snapshots of @persistent (SSH host keys, /etc/secureboot, the + # NixOS state). Same layout as the other physical hosts; no off-host target. + modules.btrbk.enable = true; + + # smartd autodetects devices and logs to the journal; the smartctl exporter + # (modules/nixos/base/monitoring.nix) is what feeds wear and media errors to + # VictoriaMetrics. + services.smartd.enable = true; + + # The inference engine ships as an OCI image and is meant to run rootless, so + # `podman` is what this host uses to build and start it from the `~/src` checkout + # that ./preservation.nix keeps. + virtualisation.podman.enable = true; + + # systemd-journal - cap disk usage + # https://www.freedesktop.org/software/systemd/man/latest/journald.conf.html + services.journald.settings.Journal = { + SystemMaxUse = "10G"; + RuntimeMaxUse = "256M"; + }; + + networking = { + inherit hostName; + + # off: this host uses networkd (below). + networkmanager.enable = false; + useDHCP = false; + }; + + # suzi's fake-ip DNS hands out AAAA records whose IPv6 path does not work: a + # `curl -6` to a registry returns 502 or times out, while `curl -4` answers in + # 0.1 s. Clients that try the AAAA first (podman, git, pip) then hang on the + # handshake, which is why docker.io, github.com and pypi.org looked blocked + # here. These are RFC 6724's default precedences with the IPv4-mapped row + # raised, so IPv4 sorts first; glibc drops its whole built-in table as soon as + # the file lists any entry, hence spelling all of it out. + # + # The real fix belongs on that gateway: stop answering AAAA for proxied names + # (`dns.ipv6: false` in mihomo; the module README says the same). + networking.getaddrinfo.precedence = { + "::1/128" = 50; + "::/0" = 40; + "2002::/16" = 30; + "::/96" = 20; + "::ffff:0:0/96" = 100; + }; + + # A `boot.kernel.sysctl` that disabled IPv6 was removed on 2026-10-14: networkd + # re-enables it on the wired link for the `DHCP = "ipv6"`/`IPv6AcceptRA` settings + # below, so it never applied and the AAAA attempt already fails at once (WA-034). + + networking.useNetworkd = true; + systemd.network.enable = true; + + # Predictable names follow the PCIe topology, so match the `en*` prefix instead of + # a name; one wired port, so one link. + systemd.network.networks."10-lan" = { + matchConfig.Name = [ "en*" ]; + networkConfig = { + Address = [ + ipv4WithMask + ]; + # DNS goes to the proxy gateway, which resolves through the proxy; + # `nameservers` is the plain upstream-DNS alternative. + # DNS = nameservers; + DNS = [ proxyGateway ]; + DHCP = "ipv6"; # enable DHCPv6 only, so we can get a GUA. + IPv6AcceptRA = true; # for Stateless IPv6 Autoconfiguration (SLAAC) + LinkLocalAddressing = "ipv6"; + # The RTL8125BG flaps carrier on link re-training; without this networkd + # tears the static address and route down and the host drops off the LAN. + IgnoreCarrierLoss = "10s"; + }; + routes = [ + { + Destination = "0.0.0.0/0"; + Gateway = proxyGateway; + } + { + Destination = "::/0"; + Gateway = proxyGateway6; + GatewayOnLink = true; # it's a gateway on local link. + } + ]; + linkConfig.RequiredForOnline = "routable"; + }; + + # This value determines the NixOS release from which the default + # settings for stateful data (file locations, database versions) are taken. + # It is the release this host was installed on. + system.stateVersion = "26.11"; +} diff --git a/hosts/idols-aquamarine/disko-fs.nix b/hosts/idols-aquamarine/disko-fs.nix new file mode 100644 index 00000000..34ab95fc --- /dev/null +++ b/hosts/idols-aquamarine/disko-fs.nix @@ -0,0 +1,132 @@ +# Disko layout for idols-aquamarine's root disk: the Fanxiang S790 2TB that was +# `ai`'s /data disk. Same shape as hosts/idols-ai/disko-fs.nix +# (ESP + LUKS + btrfs subvolumes, ephemeral tmpfs root, preservation on +# /persistent), so the two hosts share one layout and one restore procedure. +# +# The LUKS mapper is named `aqua-luks`, not `nixos-luks`: `ai` already owns that +# name while this disk is formatted (./README.md). disko generates the matching +# boot.initrd.luks entry. +# +# Destroy, format & mount as root (wipes the disk; run from ~/nix-config, and +# check `lsblk` + the by-id path first — see hosts/idols-aquamarine/README.md): +# sudo nix run github:nix-community/disko -- --mode destroy,format,mount hosts/idols-aquamarine/disko-fs.nix +# Mount only (after the first format): +# sudo nix run github:nix-community/disko -- --mode mount hosts/idols-aquamarine/disko-fs.nix +{ + # Ephemeral root; preservation mounts /persistent for state. + fileSystems."/persistent".neededForBoot = true; + + disko.devices = { + # Ephemeral root; relatime and mode=755 so systemd does not set 777. + nodev."/" = { + fsType = "tmpfs"; + mountOptions = [ + "nosuid" # Ignore setuid bit; no suid binaries live on the root tmpfs + "nodev" # Ignore device files; nothing here needs to be a device node + "size=4G" + "relatime" # Update inode access times relative to modify/change time + "mode=755" + ]; + }; + + disk.nixos-aquamarine = { + type = "disk"; + device = "/dev/disk/by-id/nvme-Fanxiang_S790_2TB_FXS790254050582"; + content = { + type = "gpt"; + partitions = { + # EFI system partition; must stay unencrypted for UEFI to load the bootloader. + ESP = { + priority = 1; + name = "ESP"; + start = "1M"; + end = "600M"; + type = "EF00"; # EF00 = ESP in GPT + content = { + type = "filesystem"; + format = "vfat"; + mountpoint = "/boot"; + mountOptions = [ + "fmask=0177" # File mask: 777-177=600 (owner rw-, group/others ---) + "dmask=0077" # Directory mask: 777-077=700 (owner rwx, group/others ---) + "noexec,nosuid,nodev" # Security: no execution, ignore setuid, no device nodes + ]; + }; + }; + # Root partition: LUKS encrypted, then btrfs with subvolumes. + root = { + size = "100%"; + content = { + type = "luks"; + name = "aqua-luks"; # Mapper name; match boot.initrd.luks + settings = { + allowDiscards = true; # TRIM for SSDs; slightly less secure, better performance + }; + # Add boot.initrd.luks.devices so initrd prompts for passphrase at boot + initrdUnlock = true; + # cryptsetup luksFormat options + extraFormatArgs = [ + "--type luks2" + "--cipher aes-xts-plain64" + "--hash sha512" + "--iter-time 5000" + "--key-size 256" + "--pbkdf argon2id" + "--use-random" # Block until enough entropy from /dev/random + ]; + extraOpenArgs = [ + "--timeout 10" + ]; + content = { + type = "btrfs"; + extraArgs = [ "-f" ]; # Force overwrite if filesystem already exists + subvolumes = { + # Top-level subvolume (id 5); used for btrfs send/receive and snapshots + "/" = { + mountpoint = "/btr_pool"; + mountOptions = [ "subvolid=5" ]; + }; + "@nix" = { + mountpoint = "/nix"; + mountOptions = [ + "compress-force=zstd:1" # Save space and reduce I/O on SSD + "noatime" + ]; + }; + "@persistent" = { + mountpoint = "/persistent"; + mountOptions = [ + "compress-force=zstd:1" + ]; + }; + "@snapshots" = { + mountpoint = "/snapshots"; + mountOptions = [ + "nosuid" + "nodev" + "compress-force=zstd:1" + ]; + }; + "@tmp" = { + mountpoint = "/tmp"; + mountOptions = [ + "nosuid" + "nodev" + "noatime" # Temp files don't need access times; saves btrfs metadata writes + "compress-force=zstd:1" + ]; + }; + # disko creates the 96G swapfile on this subvolume and adds swapDevices + "@swap" = { + mountpoint = "/swap"; + swap.swapfile.size = "96G"; + }; + }; + }; + }; + }; + }; + }; + }; + }; +} diff --git a/hosts/idols-aquamarine/hardware-amd-gpu.nix b/hosts/idols-aquamarine/hardware-amd-gpu.nix new file mode 100644 index 00000000..a5ff6551 --- /dev/null +++ b/hosts/idols-aquamarine/hardware-amd-gpu.nix @@ -0,0 +1,84 @@ +{ + pkgs, + myvars, + ... +}: +{ + # =============================================================================================== + # 2x AMD Radeon AI PRO R9700 (RDNA4, gfx1201) for local inference + # https://wiki.nixos.org/wiki/AMD_GPU + # + # The second full-length x16 slot is chipset-attached: its root port is Gen4 x4 + # while the card side reports x16, so the second card reads weights at ~7.1 GB/s + # against 46.3 GB/s. Split models by layer, not tensor parallel — see ./README.md + # and ./ROCM.md (ROCm support, measured links, RCCL TP=2 deadlock). + # =============================================================================================== + + boot.kernelModules = [ "amdgpu" ]; + + boot.kernelParams = [ + # ROCm: multi-GPU systems may hang in compute applications unless the IOMMU + # runs in passthrough mode. + # https://rocm.docs.amd.com/projects/install-on-linux/en/latest/reference/system-requirements.html + "iommu=pt" + + # This board's DSDT advertises ATPX power control (`\_SB.PC00.GFX0.ATPX`) but + # never defines the `PEG0.PEGP._OFF`/`_ON` symbols its body calls, so the first + # idle resume of the headless card leaves it in `runtime_status = error` until + # reboot; amdgpu discards the ACPI error, so the failure is silent. This is the + # only setting that avoids it, and it needs the iGPU kept for the console + # fallback to trigger at all. Root cause, kernel-source trail and the field + # comparison: ./ROCM.md (WA-032). + "amdgpu.runpm=0" + + # The driver's own default is -1, "auto, disabled except SRIOV", so on bare + # metal a hung card stays hung until the next reboot. Nobody is here to + # power-cycle an unattended inference host, so switch the recovery path on. + "amdgpu.gpu_recovery=1" + + # ASPM L0s/L1 on a link that carries PCIe traffic for a compute card buys + # little and is a known source of link retraining and timeouts; this repo + # already falls back to the same knob for NVMe APST problems (WA-008). + "pcie_aspm=off" + + # Keep the CPU out of the deepest C-states. Inference waits on short PCIe and + # memory round-trips, and the exit latency from C6/C7 shows up as decode + # jitter. Costs idle power: drop this line if that matters more than tail + # latency. + "processor.max_cstate=2" + ]; + + # amdgpu firmware. + hardware.enableRedistributableFirmware = true; + + hardware.graphics = { + enable = true; + }; + + # The ROCm OpenCL ICD, which is also what HIP consumers resolve for compute. + hardware.amdgpu.opencl.enable = true; + + # /dev/kfd is world-writable under nixpkgs' default udev rule, but + # /dev/dri/renderD* is `render` 0660 and this headless host has no logged-in + # seat to hand out ACLs, so the inference user needs the group. + users.users.${myvars.username}.extraGroups = [ "render" ]; + + environment.systemPackages = [ + pkgs.rocmPackages.rocm-smi + pkgs.rocmPackages.rocminfo + pkgs.nvtopPackages.full + ]; + + # GPU panel in btop, and utilization/VRAM in node_exporter for the shared + # VictoriaMetrics scrape — the same shape as the Radeon iGPU hosts. + # perfmon: CAP_PERFMON is what btop needs for the i915 iGPU. + modules.btop = { + package = pkgs.btop-rocm; + perfmon = true; + }; + # `enabledCollectors` is a list option, so this merges with the base module's + # `[ "systemd" "logind" ]` instead of replacing it: only name the collector + # this host adds. Repeating the base entries passes `--collector.systemd` + # twice, which node_exporter rejects, and the unit restart-loops. + services.prometheus.exporters.node.enabledCollectors = [ "drm" ]; +} diff --git a/hosts/idols-aquamarine/hardware-configuration.nix b/hosts/idols-aquamarine/hardware-configuration.nix new file mode 100644 index 00000000..66e757c7 --- /dev/null +++ b/hosts/idols-aquamarine/hardware-configuration.nix @@ -0,0 +1,73 @@ +# Hand-adapted from the `nixos-generate-config --show-hardware-config` scan that +# `ai` captured on this same Colorful CVN Z890 ARK FROZEN V20 board, with the +# KINGBANK-specific parts left behind. The +# mounts themselves are declared in ./disko-fs.nix, so nothing here names a +# disk. Regenerate this file on the target (`nixos-generate-config --root /mnt`) +# if the hardware scan ever differs. +{ + config, + lib, + pkgs, + modulesPath, + ... +}: +{ + imports = [ + (modulesPath + "/installer/scan/not-detected.nix") + ]; + + boot.kernelParams = [ + # Carried over from `ai`, where it caps NVMe APST wake latency for the + # DRAM-less KINGBANK KP260 (WA-008 in WORKAROUNDS.md). This disk (Fanxiang + # S790) has shown no APST problem; the cap stays as idle-power insurance. + "nvme_core.default_ps_max_latency_us=100000" + ]; + + boot.loader.efi.canTouchEfiVariables = true; + boot.loader.efi.efiSysMountPoint = "/boot"; + + # The nixpkgs default (`linuxPackages`) is the stable/LTS line, which is what a + # server should run; `linuxPackages_latest` is for chasing new hardware. The + # R9700 (Navi 48) only needs the 6.14 AMD validates for it, so the default is + # far ahead of the requirement. See ./ROCM.md. + # https://github.com/nixpkgs/nixpkgs/blob/nixos-unstable/pkgs/top-level/linux-kernels.nix + boot.kernelPackages = pkgs.linuxPackages; + + boot.initrd.availableKernelModules = [ + "xhci_pci" + "ahci" + "nvme" + "usbhid" + "usb_storage" + "sd_mod" + ]; + # clear /tmp on boot to get a stateless /tmp directory. + boot.tmp.cleanOnBoot = true; + + # Enable binfmt emulation of aarch64-linux, this is required for cross compilation. + boot.binfmt.emulatedSystems = [ + "aarch64-linux" + "riscv64-linux" + ]; + # This enables the kernel to preload the emulator binaries when the binfmt registrations are added, + # obviating the need to make the emulator binaries available inside chroots and chroot-like sandboxes. + boot.binfmt.preferStaticEmulators = true; # required to work with podman + + # supported file systems, so we can mount any removable disks with these filesystems + boot.supportedFilesystems = [ + "ext4" + "btrfs" + "xfs" + "ntfs" + "fat" + "vfat" + "exfat" + ]; + + # LUKS initrd, all fileSystems (/, /boot, /btr_pool, /nix, /persistent, /snapshots, /tmp, /swap) + # and swap (including /swap/swapfile bind and swapDevices) are managed by disko (disko-fs.nix). + + nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux"; + powerManagement.cpuFreqGovernor = lib.mkDefault "performance"; # ondemand / powersave / performance + hardware.cpu.intel.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware; +} diff --git a/hosts/idols-aquamarine/preservation.nix b/hosts/idols-aquamarine/preservation.nix new file mode 100644 index 00000000..d021764e --- /dev/null +++ b/hosts/idols-aquamarine/preservation.nix @@ -0,0 +1,110 @@ +{ + preservation, + lib, + pkgs, + myvars, + ... +}: +let + inherit (myvars) username; +in +{ + imports = [ + preservation.nixosModules.default + ]; + + preservation.enable = true; + # preservation requires a systemd initrd. + boot.initrd.systemd.enable = true; + + environment.systemPackages = [ + # Whole-filesystem disk usage (`-x` stays on one filesystem): + # sudo gdu -n -x / # non-interactive, plain text + pkgs.gdu + ]; + + # NOTE: preservation only mounts the directory/file list below to /persistent + # If the directory/file already exists in the root filesystem you should + # move those files/directories to /persistent first! + preservation.preserveAt."/persistent" = { + directories = [ + "/etc/ssh" + "/etc/nix/inputs" + "/etc/secureboot" # lanzaboote - secure boot (see ./secureboot.nix) + + "/var/log" + "/var/lib" + "/var/cache" + # Re-creatable scratch, but it must not consume the root tmpfs (RAM), so + # it is bind-mounted onto the persistent volume and wiped at boot below. + # The mode must stay 1777 to match that `D!` rule. + { + directory = "/var/tmp"; + mode = "1777"; + mountOptions = [ + "nosuid" + "nodev" + ]; + } + ]; + files = [ + # auto-generated machine ID + { + file = "/etc/machine-id"; + inInitrd = true; + } + ]; + + # the following directories will be passed to /persistent/home/$USER + users.${username} = { + directories = [ + # Rootless Podman's image and layer store. On the root tmpfs the 4 GiB root + # fills up during the first build, so it lives on the persistent volume + # like it does on `ai`. + ".local/share/containers" + "codes" + # Third-party source checkouts, the same layout `ai` uses: Strata, the + # inference engine whose image this host builds, lives at ~/src/Strata. + "src" + "nix-config" + "tmp" + # Model weights live here. The root filesystem is tmpfs, so anything + # not listed is gone on the next boot; weights are re-downloadable, but + # re-downloading tens of GB is not a boot step. + "models" + ]; + }; + }; + + # `.config`/`.local` are not immediate parents of any persisted path here, so + # tmpfiles would create them root:root 0755 and the user could not write + # under them. Same fix as hosts/idols-ai/preservation.nix. + systemd.tmpfiles.settings.preservation = { + "/home/${username}/.config".d = { + user = username; + group = lib.mkForce username; + mode = lib.mkForce "0750"; + }; + "/home/${username}/.local".d = { + user = username; + group = lib.mkForce username; + mode = lib.mkForce "0750"; + }; + "/home/${username}/.local/share".d = { + user = username; + group = lib.mkForce username; + mode = lib.mkForce "0750"; + }; + }; + + systemd.tmpfiles.rules = [ + # /var/tmp only accumulates between reboots (mirrors `boot.tmp.cleanOnBoot` + # for /tmp). Crash dumps can hold secrets copied out of process memory. + "D! /var/tmp 1777 root root" + "D! /var/lib/systemd/coredump 0755 root root" + ]; + + # Mask the commit service: /etc/machine-id is persisted by preservation + # (see the `files` entry above), so there is no transient ID to commit. + systemd.suppressedSystemUnits = [ "systemd-machine-id-commit.service" ]; +} diff --git a/hosts/idols-aquamarine/secureboot.nix b/hosts/idols-aquamarine/secureboot.nix new file mode 100644 index 00000000..9547e4e2 --- /dev/null +++ b/hosts/idols-aquamarine/secureboot.nix @@ -0,0 +1,30 @@ +{ + pkgs, + lib, + lanzaboote, + ... +}: +{ + # Secure Boot here reuses the PKI already enrolled on the Z890 board, carried over with the disk + # (./README.md step 3). lanzaboote signs with `pkiBundle` below, so a fresh key set must end up in + # /etc/secureboot — `sbctl create-keys` writes to its own keydir — or the firmware trusts a key the + # UKI was not signed with. New keys also change PCR7, which invalidates a TPM2 LUKS enrollment made + # against the old ones (./README.md step 4). + imports = [ + lanzaboote.nixosModules.lanzaboote + ]; + + environment.systemPackages = [ + # For debugging and troubleshooting Secure Boot. + pkgs.sbctl + ]; + + # Lanzaboote replaces the systemd-boot module. The generated + # hardware-configuration.nix sets this to true, so force it off. + boot.loader.systemd-boot.enable = lib.mkForce false; + + boot.lanzaboote = { + enable = true; + pkiBundle = "/etc/secureboot"; + }; +} diff --git a/nixos-installer/README.md b/nixos-installer/README.md index d6531879..a471ed2b 100644 --- a/nixos-installer/README.md +++ b/nixos-installer/README.md @@ -48,8 +48,8 @@ nix run github:nix-community/disko -- --mode destroy,format,mount ../hosts/idols # Mount only (e.g. after first format, without wiping): # nix run github:nix-community/disko -- --mode mount ../hosts/idols-ai/disko-fs.nix -# the separate data disk (mounts at /data): -nix run github:nix-community/disko -- --mode destroy,format,mount ../hosts/idols-ai/disko-fs-data.nix +# Other hosts keep their own layout file. The inference server's root disk is installed from a +# running host, not from this ISO flow: see ../hosts/idols-aquamarine/README.md for the order. # setup the automatic unlock via the tpm2 chip # find the LUKS partition with: lsblk -f diff --git a/outputs/README.md b/outputs/README.md index e9e6cb01..5d5c6118 100644 --- a/outputs/README.md +++ b/outputs/README.md @@ -47,10 +47,10 @@ Some tests iterate over every configuration, so a new host is covered automatica them. Others name specific hosts — or a name prefix, like `k3s-api-endpoint`, which covers every `k3s-test-1-*` configuration — and only cover a new host if you add it. -| Kind | Tests | -| ------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| Every configuration | `hostname`, `kernel`, `nix-system-features`, `security-apparmor`, `security-container-groups`, `security-exporters`, `security-firewall`, `security-k3s-kubeconfig`, `security-k3s-pod-firewall`, `security-kernel`, `security-ssh-x11` | -| Named hosts only | `btrbk`, `home-manager`, `home-manager-xdg`, `computer-use-headless`, `k3s-api-endpoint`, `k3s-node-home-manager`, `idols-ai-gpu`, `idols-ai-voice-input`, `security-postgres-auth`, `youko-metrics`, `ups-metrics`, `shoukei-logind` | +| Kind | Tests | +| ------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Every configuration | `hostname`, `kernel`, `nix-system-features`, `security-apparmor`, `security-container-groups`, `security-exporters`, `security-firewall`, `security-k3s-kubeconfig`, `security-k3s-pod-firewall`, `security-kernel`, `security-ssh-x11` | +| Named hosts only | `btrbk`, `home-manager`, `home-manager-xdg`, `computer-use-headless`, `k3s-api-endpoint`, `k3s-node-home-manager`, `idols-ai-gpu`, `idols-ai-voice-input`, `idols-aquamarine`, `security-postgres-auth`, `youko-metrics`, `ups-metrics`, `shoukei-logind` | `hostname` expects each configuration's `networking.hostName` to equal its name. Niri desktop configurations are the exception (`ai-niri` → `ai`, `shoukei-niri` → `shoukei`) and are listed in diff --git a/outputs/x86_64-linux/src/idols-aquamarine.nix b/outputs/x86_64-linux/src/idols-aquamarine.nix new file mode 100644 index 00000000..969e62a5 --- /dev/null +++ b/outputs/x86_64-linux/src/idols-aquamarine.nix @@ -0,0 +1,50 @@ +{ + # NOTE: the args not used in this file CAN NOT be removed! + # because haumea pass argument lazily, + # and these arguments are used in the functions like `mylib.nixosSystem`, `mylib.colmenaSystem`, etc. + inputs, + lib, + mylib, + myvars, + system, + genSpecialArgs, + ... +}@args: +let + # 星野 愛久愛海, Hoshino Akuamarin + name = "aquamarine"; + tags = [ + name + "homelab-ai" + ]; + ssh-user = "root"; + + modules = { + nixos-modules = + (map mylib.relativeToRoot [ + # common + "modules/nixos/server/server.nix" + # host specific + "hosts/idols-${name}" + ]) + ++ [ + # No `modules.secrets.server.*` group yet: the host would fail activation + # until its new host key is a recipient in nix-secrets. Enabling one is + # the last step in hosts/idols-aquamarine/README.md. + ]; + }; + + systemArgs = modules // args; +in +{ + nixosConfigurations.${name} = mylib.nixosSystem systemArgs; + + colmena.${name} = mylib.colmenaSystem ( + systemArgs + // { + inherit tags ssh-user; + } + ); + + packages.${name} = inputs.self.nixosConfigurations.${name}.config.system.build.images.iso; +} diff --git a/outputs/x86_64-linux/tests/idols-aquamarine/expected.nix b/outputs/x86_64-linux/tests/idols-aquamarine/expected.nix new file mode 100644 index 00000000..86397f33 --- /dev/null +++ b/outputs/x86_64-linux/tests/idols-aquamarine/expected.nix @@ -0,0 +1,23 @@ +{ + hostNameIsBare = true; + amdgpuDriverLoaded = true; + iommuPassthrough = true; + runtimePmDisabled = true; + gpuRecoveryEnabled = true; + aspmDisabled = true; + cStatesCapped = true; + prefersIpv4 = true; + sourceCheckoutsPersistent = true; + rootlessPodmanEnabled = true; + stableKernel = true; + btopShowsRadeon = true; + btopReadsIgpu = true; + gpuMetricsExported = true; + collectorFlagsUnique = true; + inferenceUserCanRender = true; + noHomeManager = true; + staticAddressMatchesWiredInterface = true; + rootIsTmpfs = true; + stateIsPersistent = true; + luksMapperNamed = true; +} diff --git a/outputs/x86_64-linux/tests/idols-aquamarine/expr.nix b/outputs/x86_64-linux/tests/idols-aquamarine/expr.nix new file mode 100644 index 00000000..aec1735d --- /dev/null +++ b/outputs/x86_64-linux/tests/idols-aquamarine/expr.nix @@ -0,0 +1,67 @@ +{ + myvars, + outputs, + lib, + ... +}: +let + eval = outputs.nixosConfigurations.aquamarine; + cfg = eval.config; + inherit (myvars.networking.hostsAddr.aquamarine) ipv4; +in +{ + # A server configuration is named after the bare hostname (no `-niri`). + hostNameIsBare = cfg.networking.hostName == "aquamarine"; + + # The two Radeon AI PRO R9700 cards: driver and the IOMMU mode ROCm needs for multi-GPU. + amdgpuDriverLoaded = builtins.elem "amdgpu" cfg.boot.kernelModules; + iommuPassthrough = builtins.elem "iommu=pt" cfg.boot.kernelParams; + # This board's ACPI firmware cannot power the dGPU back on, and amdgpu throws + # the error away, so the card dies on its first idle resume (./ROCM.md, WA-032). + runtimePmDisabled = builtins.elem "amdgpu.runpm=0" cfg.boot.kernelParams; + + # An unattended host recovers a hung card itself instead of waiting for a reboot. + gpuRecoveryEnabled = builtins.elem "amdgpu.gpu_recovery=1" cfg.boot.kernelParams; + aspmDisabled = builtins.elem "pcie_aspm=off" cfg.boot.kernelParams; + cStatesCapped = builtins.elem "processor.max_cstate=2" cfg.boot.kernelParams; + + # suzi's fake-ip DNS returns AAAA records with no working IPv6 path, so the + # generated gai.conf has to sort IPv4 first (see ./default.nix). + prefersIpv4 = lib.hasInfix "precedence ::ffff:0:0/96 100" cfg.environment.etc."gai.conf".text; + + # The Strata checkout lives under ~/src, which has to survive the tmpfs root + # like ~/codes does. + sourceCheckoutsPersistent = builtins.any ( + d: d.directory == "/home/${myvars.username}/src" + ) cfg.preservation.preserveAt."/persistent".users.${myvars.username}.directories; + rootlessPodmanEnabled = cfg.virtualisation.podman.enable; + + # A server tracks the nixpkgs default (stable/LTS) kernel, not `-latest`. + stableKernel = cfg.boot.kernelPackages.kernel.version == eval.pkgs.linuxPackages.kernel.version; + btopShowsRadeon = cfg.modules.btop.package == eval.pkgs.btop-rocm; + # CAP_PERFMON inside the wrapper is what the i915 iGPU needs. + btopReadsIgpu = cfg.modules.btop.perfmon; + gpuMetricsExported = builtins.elem "drm" cfg.services.prometheus.exporters.node.enabledCollectors; + # node_exporter rejects a repeated flag, and this list merges with the base + # module's entries, so a duplicate restart-loops the unit. + collectorFlagsUnique = + builtins.length (cfg.services.prometheus.exporters.node.enabledCollectors) + == builtins.length (lib.unique cfg.services.prometheus.exporters.node.enabledCollectors); + inferenceUserCanRender = builtins.elem "render" cfg.users.users.${myvars.username}.extraGroups; + + # Headless: no desktop session, no Home Manager. + noHomeManager = !cfg ? home-manager.users; + + # The board renames the wired NIC with the PCIe topology, so networkd matches the + # `en*` prefix; a name-based match leaves the host without an address. + staticAddressMatchesWiredInterface = + builtins.elem "${ipv4}/24" cfg.systemd.network.networks."10-lan".networkConfig.Address + && cfg.systemd.network.networks."10-lan".matchConfig.Name == [ "en*" ]; + + # Ephemeral root with the persistent volume required at boot, as on `ai`. + rootIsTmpfs = cfg.fileSystems."/".fsType or null == "tmpfs"; + stateIsPersistent = cfg.fileSystems."/persistent".neededForBoot or false; + + # Install step 4 of ./README.md depends on this mapper name staying put. + luksMapperNamed = builtins.attrNames cfg.boot.initrd.luks.devices == [ "aqua-luks" ]; +} diff --git a/vars/networking.nix b/vars/networking.nix index cc281402..252f30d6 100644 --- a/vars/networking.nix +++ b/vars/networking.nix @@ -50,8 +50,9 @@ rec { ipv6 = "fe80::10"; # Link-local Address }; aquamarine = { - # VM (retired) - iface = "enp2s0"; + # Physical AI inference server. Reuses the retired VM's address, so check + # that the router still reserves .101 for this machine. No `iface`: the host + # matches the wired interface by its `en*` prefix (hosts/idols-aquamarine/). ipv4 = "192.168.5.101"; }; ruby = { @@ -202,7 +203,9 @@ rec { { # Define the root user's host key for remote builders, so that nix can verify all the remote builders - aquamarine.publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIEOXFhFu9Duzp6ZBE288gDZ6VLrNaeWL4kDrFUh9Neic root@aquamarine"; + # Reinstalled on 2026-10-13; this is the host key its first boot + # generated (hosts/idols-aquamarine/README.md). + aquamarine.publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHeZ1eTvzcoOOSmRCto/v9hKfirzaGh6/7bl5I2ly3Hj root@aquamarine"; # the restic backup server youko.publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIEs8b3CmxM+utXZVKvJBssXBupBXvvzjHP40HcY7PSyZ root@youko"; # ruby.publicKey = "";