diff --git a/WORKAROUNDS.md b/WORKAROUNDS.md index 0f88c755..274b4528 100644 --- a/WORKAROUNDS.md +++ b/WORKAROUNDS.md @@ -66,14 +66,15 @@ only delete inactive duplicates. A known limitation with no fix yet. -| ID | What & where | Why | Removal condition | Added | Revisit | Status | -| ------ | ---------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ----------------------------------------------------------------------------------------------------------------------------- | ---------- | -------------------------------------------------------------- | ------ | -| WA-016 | subscription rule snapshots — `modules/nixos/desktop/networking/mihomo/` (`rules_from:`) | mihomo's `proxy-providers` imports proxies only, so the `rules` and `proxy-groups` a subscription ships with are dropped by the core; the generator re-imports them from a local YAML copy that has to be refreshed by hand when the provider changes its routing policy | Drop the snapshots once mihomo can import rules from a proxy-provider, or the providers publish their rules as rule-providers | 2026-10-04 | when mihomo gains rule import | active | -| WA-022 | qemu-user binfmt removed — `hosts/12kingdoms-shoukei/hardware-configuration.nix` | the x86-on-arm module owns the x86 ELF handlers and this host does not need riscv64, but dropping `boot.binfmt.emulatedSystems` also drops `nix.settings.extra-platforms` and qemu-user, so `podman run --platform=linux/amd64`, x86 chroots and local builds of uncached x86 derivations need a remote x86_64 builder | Restore `boot.binfmt.emulatedSystems` (without x86_64/i686, which the module asserts against) if such a workload is needed | 2026-10-07 | when an amd64 container/chroot or uncached x86 build is needed | active | -| WA-025 | brcmfmac 4-way-handshake offload authentication — upstream kernel/wpa_supplicant, no code in this repo | Since hostap commit 416386060, wpa_supplicant 2.11+ expects `NL80211_CMD_PORT_AUTHORIZED` from drivers that offload the 4-way handshake; brcmfmac never reports it, so authentication intermittently fails on Apple Silicon (BRCM4378/4387/4388). nixpkgs already carries the Broadcom revert in wpa_supplicant 2.12; the proper fix, Janne Grunau's `wifi: brcmfmac: cfg80211: Report port_authorized for 4-way HS offload` (2026-09), is still under review. Entering the key again after such a failure only creates a duplicate profile, so cancel and retry instead | Fix lands via a nixpkgs kernel update (or by carrying the patch through `nixpkgs-patched`) | 2026-10-07 | on every kernel bump | active | -| WA-033 | USB-C/XHCI port `_PLD` ACPI noise — this board's DSDT, no code in this repo | every boot logs 16 `ACPI Error: Aborting method \_SB.PC00.TXHC.RHUB.SS01._PLD ... (AE_NOT_FOUND)` lines, plus `\_SB.PC02.XHCI.RHUB.HS01._PLD`; the ports work, and these messages are unrelated to the GPU's ATPX failure, which names `\_SB.PC00.GFX0.ATPX` instead | Drop if a BIOS update defines the `_PLD` descriptors | 2026-10-14 | on a BIOS update | active | -| WA-034 | IPv6-first resolution is dead on the wired link — `hosts/idols-aquamarine/default.nix` (`networking.getaddrinfo.precedence`) | the proxy gateway answers DNS in fake-IP mode and its IPv6 path is dead: an AAAA lookup returns a fake address whose connection fails at once (measured: `curl -6` fails in 8 ms, `curl -4` answers in 0.7 s). glibc clients are covered by the RFC 6724 table with the IPv4-mapped row raised; nix's libcurl and other non-glibc resolvers ignore `/etc/gai.conf`. Disabling IPv6 with `boot.kernel.sysctl` did not work here — networkd re-enables it on the wired link for `DHCP = "ipv6"`/`IPv6AcceptRA` — so that setting was dropped on 2026-10-14 | Drop once the gateway serves proxied names over IPv6 as well (the adjacent fix is `dns.ipv6: false` in mihomo, see WA-030) | 2026-10-14 | on a mihomo or gateway change | active | -| WA-035 | SSH retry + multiplexing to LAN peers, ARP fallback tuning — `home/base/tui/ssh.nix`, `hosts/idols-ai/neighbour-probe.nix` | the desktop reaches the LAN over a Mesh WiFi backhaul, which drops unicast to a peer whose MAC it has aged out of its forwarding table while still flooding broadcast: a capture shows seven SYNs from one socket addressed to the cached MAC with no answer, three unicast ARP probes unanswered, then a broadcast ARP answered in 0.4 ms after ~12 s of failed connects. The ssh settings retry a short connect and keep a master warm so the entry stays fresh; the neighbour-probe tunings walk the kernel to that broadcast fallback in ~1 s | Drop once the backhaul is wired or the mesh stops dropping unknown unicast | 2026-10-16 | when the backhaul changes | active | +| ID | What & where | Why | Removal condition | Added | Revisit | Status | +| ------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ----------------------------------------------------------------------------------------------------------------------------- | ---------- | -------------------------------------------------------------- | ------ | +| WA-016 | subscription rule snapshots — `modules/nixos/desktop/networking/mihomo/` (`rules_from:`) | mihomo's `proxy-providers` imports proxies only, so the `rules` and `proxy-groups` a subscription ships with are dropped by the core; the generator re-imports them from a local YAML copy that has to be refreshed by hand when the provider changes its routing policy | Drop the snapshots once mihomo can import rules from a proxy-provider, or the providers publish their rules as rule-providers | 2026-10-04 | when mihomo gains rule import | active | +| WA-022 | qemu-user binfmt removed — `hosts/12kingdoms-shoukei/hardware-configuration.nix` | the x86-on-arm module owns the x86 ELF handlers and this host does not need riscv64, but dropping `boot.binfmt.emulatedSystems` also drops `nix.settings.extra-platforms` and qemu-user, so `podman run --platform=linux/amd64`, x86 chroots and local builds of uncached x86 derivations need a remote x86_64 builder | Restore `boot.binfmt.emulatedSystems` (without x86_64/i686, which the module asserts against) if such a workload is needed | 2026-10-07 | when an amd64 container/chroot or uncached x86 build is needed | active | +| WA-025 | brcmfmac 4-way-handshake offload authentication — upstream kernel/wpa_supplicant, no code in this repo | Since hostap commit 416386060, wpa_supplicant 2.11+ expects `NL80211_CMD_PORT_AUTHORIZED` from drivers that offload the 4-way handshake; brcmfmac never reports it, so authentication intermittently fails on Apple Silicon (BRCM4378/4387/4388). nixpkgs already carries the Broadcom revert in wpa_supplicant 2.12; the proper fix, Janne Grunau's `wifi: brcmfmac: cfg80211: Report port_authorized for 4-way HS offload` (2026-09), is still under review. Entering the key again after such a failure only creates a duplicate profile, so cancel and retry instead | Fix lands via a nixpkgs kernel update (or by carrying the patch through `nixpkgs-patched`) | 2026-10-07 | on every kernel bump | active | +| WA-033 | USB-C/XHCI port `_PLD` ACPI noise — this board's DSDT, no code in this repo | every boot logs 16 `ACPI Error: Aborting method \_SB.PC00.TXHC.RHUB.SS01._PLD ... (AE_NOT_FOUND)` lines, plus `\_SB.PC02.XHCI.RHUB.HS01._PLD`; the ports work, and these messages are unrelated to the GPU's ATPX failure, which names `\_SB.PC00.GFX0.ATPX` instead | Drop if a BIOS update defines the `_PLD` descriptors | 2026-10-14 | on a BIOS update | active | +| WA-034 | IPv6-first resolution is dead on the wired link — `hosts/idols-aquamarine/default.nix` (`networking.getaddrinfo.precedence`) | the proxy gateway answers DNS in fake-IP mode and its IPv6 path is dead: an AAAA lookup returns a fake address whose connection fails at once (measured: `curl -6` fails in 8 ms, `curl -4` answers in 0.7 s). glibc clients are covered by the RFC 6724 table with the IPv4-mapped row raised; nix's libcurl and other non-glibc resolvers ignore `/etc/gai.conf`. Disabling IPv6 with `boot.kernel.sysctl` did not work here — networkd re-enables it on the wired link for `DHCP = "ipv6"`/`IPv6AcceptRA` — so that setting was dropped on 2026-10-14 | Drop once the gateway serves proxied names over IPv6 as well (the adjacent fix is `dns.ipv6: false` in mihomo, see WA-030) | 2026-10-14 | on a mihomo or gateway change | active | +| WA-035 | SSH retry + multiplexing to LAN peers, ARP fallback tuning — `home/base/tui/ssh.nix`, `hosts/idols-ai/neighbour-probe.nix` | the desktop reaches the LAN over a Mesh WiFi backhaul, which drops unicast to a peer whose MAC it has aged out of its forwarding table while still flooding broadcast: a capture shows seven SYNs from one socket addressed to the cached MAC with no answer, three unicast ARP probes unanswered, then a broadcast ARP answered in 0.4 ms after ~12 s of failed connects. The ssh settings retry a short connect and keep a master warm so the entry stays fresh; the neighbour-probe tunings walk the kernel to that broadcast fallback in ~1 s | Drop once the backhaul is wired or the mesh stops dropping unknown unicast | 2026-10-16 | when the backhaul changes | active | +| WA-038 | OpenVINO GPU plugin needs an `ocl-icd` loader — [hosts/idols-aquamarine/hardware-intel-igpu.nix](./hosts/idols-aquamarine/hardware-intel-igpu.nix) (`openvino-gpu` wrapper) | `/run/opengl-driver/lib/libOpenCL.so.1` comes from ROCm's clr, which provides OpenCL 2.2 symbols, while OpenVINO's GPU plugin links the OpenCL 3.0 loader (`ocl-icd`): the plugin fails to load and OpenVINO silently omits GPU from `available_devices`. The wrapper runs a command with `ocl-icd` first on `LD_LIBRARY_PATH` and `OCL_ICD_VENDORS` pinned to the Intel ICD | Drop once nixpkgs' openvino GPU plugin carries an explicit `ocl-icd` RUNPATH, or ROCm's clr ships an OpenCL 3.0 loader | 2026-10-18 | on an openvino or ROCm bump | active | ## WIP diff --git a/hosts/idols-aquamarine/IGPU.md b/hosts/idols-aquamarine/IGPU.md new file mode 100644 index 00000000..d3e57653 --- /dev/null +++ b/hosts/idols-aquamarine/IGPU.md @@ -0,0 +1,135 @@ +# The Intel iGPU on aquamarine + +The Arrow Lake-S iGPU (i915) does this host's video work and can run OpenVINO models; the R9700s +stay on inference. [hardware-intel-igpu.nix](./hardware-intel-igpu.nix) installs the stack and the +`openvino-gpu` wrapper. Measured 2026-10-18, OpenVINO 2026.4.0 / openvino-genai 2026.4.0.0. + +## Devices + +`openvino-gpu` puts the OpenCL 3.0 loader (`ocl-icd`) ahead of ROCm's clr and pins the Intel ICD; +without it the GPU plugin fails to load and GPU is missing from the device list (WA-038). Use it for +every OpenVINO run: + +```bash +openvino-gpu python -c 'import openvino as ov; c = ov.Core(); print(c.available_devices)' +# ['CPU', 'GPU', 'NPU'] +openvino-gpu python -c 'import openvino as ov; print(ov.Core().get_property("GPU", "FULL_DEVICE_NAME"))' +# Intel(R) Graphics (iGPU) +``` + +VA-API uses `/dev/dri/renderD128`: + +```bash +nix shell nixpkgs#libva-utils -c vainfo --display drm --device /dev/dri/renderD128 +nix shell nixpkgs#ffmpeg -c ffmpeg -hwaccel vaapi -hwaccel_device /dev/dri/renderD128 \ + -i in.mkv -c:v hevc_vaapi -qp 26 out.mkv +``` + +## Running OpenVINO GenAI models + +Models with static or bounded-dynamic shapes compile on `GPU`; unbounded dynamic shapes fail with +`dynamic shape without upper bound`. Inspect a model before choosing the device: + +```bash +openvino-gpu python - /path/to/model.xml <<'PY' +import sys +import openvino as ov + +for port in ov.Core().read_model(sys.argv[1]).inputs: + print(port.get_any_name(), port.get_partial_shape()) +PY +``` + +### Whisper needs a static encoder + +The published `whisper-large-v3-turbo-int8-ov` IRs are fully dynamic. The NPU pipeline reshapes them +internally and the CPU path runs them as-is, but the GPU path cannot compile the encoder. Reshape it +once, with batch bounded `0..1` so OpenVINO GenAI's `{0, 128, 3000}` input reset still matches, then +run the normal pipeline: + +```bash +# Store paths change on every rebuild, so read them from the running service. +MODEL=$(systemctl cat whisper-asr | sed -n 's/^Environment="WHISPER_MODEL_DIR=\([^"]*\)"/\1/p') +ENVPY=$(systemctl cat whisper-asr | sed -n 's/^ExecStart=\([^ ]*\).*/\1/p') +OUT=/tmp/whisper-static + +rm -rf "$OUT" && mkdir -p "$OUT" +cp "$MODEL"/*.json "$OUT"/ +for f in "$MODEL"/*.bin "$MODEL"/*.xml; do + case "$(basename "$f")" in + openvino_encoder_model.*) ;; + *) ln -s "$f" "$OUT/$(basename "$f")" ;; + esac +done + +openvino-gpu "$ENVPY" - <" +print(pipe.generate(samples, config).texts[0].strip()) +PY +``` + +## Timings and memory + +Steady state, pinned language, 3-5 iterations after warmup. NPU through the `whisper-asr` service; +iGPU and CPU standalone. Audio: 11.0 s English, 5.6 s Chinese. + +| Device | English | Chinese | CPU-s/request | Memory in use | +| ------ | ------------------ | ------- | ------------- | -------------------- | +| NPU | 1.47 s (1.46-1.57) | 1.36 s | 0.055 / 0.042 | ~2.2 GiB (resident) | +| iGPU | 1.49 s | 1.41 s | 0.23 / 0.14 | ~2.1 GiB (on demand) | +| CPU | 1.23 s | 1.18 s | 9.35 / 8.99 | ~1.1 GiB (on demand) | + +| Device | Cold compile / first use | Peak memory | +| ------ | ----------------------------------------------------------------------------------- | ----------- | +| NPU | ~70 s VPUX compile on a cold `/var/cache/whisper-asr`; warm starts import the blobs | 6.2 GB | +| iGPU | load 2.2 s instead of 0.7 s warm; first inference 1.8 s instead of 1.5 s | 2.65 GB | +| CPU | load 0.95 s; first inference 1.85 s instead of 1.23 s | ~2.0 GB | + +## Language switching + +Pinned and auto-detect (no `language`) both work on the iGPU and CPU: alternating en/zh in one +process or service is correct, 1.44-1.79 s on iGPU, 1.20-1.34 s on CPU. On the NPU pinned works +(1.46/1.36 s); auto-detect works while the idle pipeline is fresh (>=4 s between requests), but an +auto request right after a pinned one inherited the pinned language and translated the English audio +to Chinese, correct again after the ~4 s rebuild. A pinned request during that rebuild took 5.25 s. +Pin the language per client before mixing modes. + +## Cache + +A `CACHE_DIR` written by a failed GPU compile produced repeated text; deleting it and recompiling +fixed the output. Wipe the cache after changing the model or after a failed attempt. + +## Which device + +- **Whisper/ASR: NPU.** Same latency as the iGPU, lowest CPU cost, leaves the iGPU free. +- **iGPU: OpenVINO models that compile on `GPU`, and ASR fallback.** It occupies the GPU for the + whole request, so transcodes and the console wait. +- **CPU: emergency only.** Fastest single request, but it keeps ~7.5 cores busy. diff --git a/hosts/idols-aquamarine/README.md b/hosts/idols-aquamarine/README.md index 66606cdd..51dbb505 100644 --- a/hosts/idols-aquamarine/README.md +++ b/hosts/idols-aquamarine/README.md @@ -9,7 +9,7 @@ that name was decommissioned; this is a different, physical machine that reuses | Part | Model | Notes | | ------ | ----------------------------------------------------- | ------------------------------------------------------- | | Board | Colorful CVN Z890 ARK FROZEN V20 (BIOS 1013) | moved from `ai` on 2026-10-13, so its EFI vars come too | -| CPU | Intel Core Ultra 7 270K Plus | Arrow Lake; NPU unused here | +| CPU | Intel Core Ultra 7 270K Plus | Arrow Lake; the NPU serves `whisper-asr` (./asr.nix) | | Memory | 2x48G JUHOR (96G), 4800 MT/s | the 2x16G kit stayed with `ai` | | GPU | 2x AMD Radeon AI PRO R9700, 32G each (RDNA4, gfx1201) | 64G of VRAM total | | Root | Fanxiang S790 2TB | was `ai`'s `/data` disk until 2026-10-13 | @@ -26,9 +26,14 @@ Two things about this board set the ceiling for the GPU work: R9700 has no xGMI link between cards, and a dual-R9700 report has RCCL deadlocking with vLLM TP=2 even with `NCCL_P2P_DISABLE=1`. See [ROCM.md](./ROCM.md). -The iGPU needs no configuration: the kernel binds `i915` and -`hardware.enableRedistributableFirmware` is on, so a monitor plugged into the board's HDMI/DP ports -gets the text console (`multi-user.target`, no display manager) when SSH is unreachable. +The iGPU carries display and video work: the kernel binds `i915`, and +[hardware-intel-igpu.nix](./hardware-intel-igpu.nix) installs `intel-media-driver` (VA-API), +`vpl-gpu-rt` (oneVPL/QSV) and `intel-compute-runtime` (OpenCL/Level-Zero), so transcoding can use +its media engine while the R9700s stay on inference. VA-API targets `/dev/dri/renderD128`, and +OpenVINO jobs run through `openvino-gpu `; [IGPU.md](./IGPU.md) records the wrapper, the +static-shape workaround Whisper needs on `GPU`, and the measured NPU/iGPU/CPU timings. A monitor +plugged into the board's HDMI/DP ports still gets the text console (`multi-user.target`, no display +manager) when SSH is unreachable. `iommu=pt` is set in [hardware-amd-gpu.nix](./hardware-amd-gpu.nix) because ROCm documents hangs on multi-GPU systems without IOMMU passthrough. The same file disables amdgpu runtime power management: diff --git a/hosts/idols-aquamarine/default.nix b/hosts/idols-aquamarine/default.nix index 7d3dba82..362f9355 100644 --- a/hosts/idols-aquamarine/default.nix +++ b/hosts/idols-aquamarine/default.nix @@ -28,6 +28,7 @@ in # hardware ./hardware-configuration.nix ./hardware-amd-gpu.nix + ./hardware-intel-igpu.nix ./preservation.nix ./secureboot.nix diff --git a/hosts/idols-aquamarine/hardware-intel-igpu.nix b/hosts/idols-aquamarine/hardware-intel-igpu.nix new file mode 100644 index 00000000..d1d8a477 --- /dev/null +++ b/hosts/idols-aquamarine/hardware-intel-igpu.nix @@ -0,0 +1,33 @@ +{ + pkgs, + ... +}: + +# =============================================================================================== +# Intel Arrow Lake-S iGPU (i915). The R9700s are reserved for model inference, so the iGPU takes +# video work and small OpenVINO jobs: iHD backs VA-API, oneVPL/QSV is Intel's video API, and +# intel-compute-runtime provides the OpenCL/Level Zero devices OpenVINO's GPU plugin needs. +# =============================================================================================== +{ + hardware.graphics.extraPackages = with pkgs; [ + intel-media-driver + vpl-gpu-rt + intel-compute-runtime + ]; + + # OpenVINO's GPU plugin links the OpenCL 3.0 loader (ocl-icd), but + # `/run/opengl-driver/lib/libOpenCL.so.1` comes from ROCm's clr (OpenCL 2.2), so the plugin + # fails to load and OpenVINO silently drops GPU from `available_devices`. Run OpenVINO jobs + # through this wrapper to put ocl-icd first and pin the Intel ICD (WORKAROUNDS.md, WA-038). + # See IGPU.md for running models on the iGPU. + environment.systemPackages = [ + (pkgs.writeShellApplication { + name = "openvino-gpu"; + text = '' + export LD_LIBRARY_PATH="${pkgs.ocl-icd}/lib''${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" + export OCL_ICD_VENDORS="${pkgs.intel-compute-runtime}/etc/OpenCL/vendors" + exec "$@" + ''; + }) + ]; +} diff --git a/outputs/x86_64-linux/tests/idols-aquamarine/expected.nix b/outputs/x86_64-linux/tests/idols-aquamarine/expected.nix index 8836bfa2..b170349d 100644 --- a/outputs/x86_64-linux/tests/idols-aquamarine/expected.nix +++ b/outputs/x86_64-linux/tests/idols-aquamarine/expected.nix @@ -14,6 +14,8 @@ stableKernel = true; btopShowsRadeon = true; btopReadsIgpu = true; + igpuMediaStackInstalled = true; + openvinoGpuWrapperInstalled = true; gpuMetricsExported = true; collectorFlagsUnique = true; inferenceUserCanRender = true; diff --git a/outputs/x86_64-linux/tests/idols-aquamarine/expr.nix b/outputs/x86_64-linux/tests/idols-aquamarine/expr.nix index aea37fd0..d43aa5e8 100644 --- a/outputs/x86_64-linux/tests/idols-aquamarine/expr.nix +++ b/outputs/x86_64-linux/tests/idols-aquamarine/expr.nix @@ -58,6 +58,20 @@ in btopShowsRadeon = cfg.modules.btop.package == eval.pkgs.btop-rocm; # CAP_PERFMON inside the wrapper is what the i915 iGPU needs. btopReadsIgpu = cfg.modules.btop.perfmon; + # The iGPU takes video work off the R9700s: VA-API, oneVPL and OpenCL/Level-Zero. + igpuMediaStackInstalled = + let + packageNames = map (package: package.pname or "") (cfg.hardware.graphics.extraPackages or [ ]); + in + builtins.all (name: builtins.elem name packageNames) [ + "intel-media-driver" + "vpl-gpu-rt" + "intel-compute-runtime" + ]; + # OpenVINO's GPU plugin needs ocl-icd ahead of ROCm's clr loader (WA-038). + openvinoGpuWrapperInstalled = builtins.any ( + package: (package.name or "") == "openvino-gpu" + ) cfg.environment.systemPackages; gpuMetricsExported = builtins.elem "drm" cfg.services.prometheus.exporters.node.enabledCollectors; # node_exporter rejects a repeated flag, and this list merges with the base # module's entries, so a duplicate restart-loops the unit.