feat(aquamarine): serve Whisper transcription from the NPU (#472)

The goal is speech recognition on the NPU for vinput, and sherpa-onnx cannot
deliver it: every model it loads keeps a dynamic time axis, which the NPU
compiler rejects outright. Qwen3-ASR and SenseVoice both core-dumped in
vpux-compiler ("Got non broadcastable dimensions pair"), and reshaping the ONNX
inputs does not help because the dynamic dimensions sit inside the graph
(WA-037). Whisper is the only family OpenVINO publishes fully static IRs for,
and its fixed 30 s window is exactly what the compiler accepts.

The host now runs an OpenAI-compatible /v1/audio/transcriptions service backed
by whisper-large-v3-turbo int8 on the NPU: 1.5 s for 11 s of audio against
2.1 s on CPU, with the compiled graph cached in /var/cache so only a cold cache
pays the ~60 s NPU compile. vinput on the desktop sends raw PCM to it through a
small provider script.

Only the NPU driver and compiler still come from the unmerged branch in the
`nixpkgs-npu` input (NixOS/nixpkgs#570230), whose nixpkgs also happens to be
the one where openvino-genai 2026.4.0.0 builds at all.

Assisted-by: DeepSeek Harness 0.2.1-rc.2 (deepseek-v4.1-flash)

hosts/idols-aquamarine/asr.nix

@@ -0,0 +1,108 @@
+{
+  lib,
+  myvars,
+  pkgs,
+  pkgs-npu,
+  ...
+}:
+let
+  inherit (myvars.networking.hostsAddr.aquamarine) ipv4;
+  port = 18080;
+
+  # OpenVINO publishes static IRs for Whisper only, and the NPU compiler accepts
+  # nothing less: any other ASR family keeps a dynamic time axis and aborts it
+  # (WORKAROUNDS.md WA-037). Pinned by content because the NPU caches the
+  # compiled graph; a version bump should be deliberate.
+  modelFiles = {
+    "added_tokens.json" = "sha256-PFH2bEwh+eEmlwB48RrnenjHSu6N9gbunauobkZxCOA=";
+    "config.json" = "sha256-v9ksCXVHqxLLQquugAi+WlmpH9xas5rM4kSJ64o+ioY=";
+    "generation_config.json" = "sha256-Rhf8ykWK87kaEDFDqqyRnBq2aAtVLXq9EIEbcki9d7Q=";
+    "merges.txt" = "sha256-LfKZCjleNejfvHUR4IwS1WAY2NBGkeATPl1jsh4VTcY=";
+    "normalizer.json" = "sha256-vxxQfchyTKnPmQNkDaz7adri8A7e5PIc66EGpzkvJt0=";
+    "openvino_config.json" = "sha256-naOOiM7Aaa1UaZ1ifxxZ02w2o9+1S14btc/fWDLvrwM=";
+    "openvino_decoder_model.bin" = "sha256-wGSZHLr8Q4FWfSmXK3AT3CQCbenDJtA+sebk/ESqlZ8=";
+    "openvino_decoder_model.xml" = "sha256-rrCfr78cDL+EuvMPRnY0NgBYIvrzs2U1m43gqgTwMEc=";
+    "openvino_detokenizer.bin" = "sha256-8rPEeCWhCJUl/2XAyOSSceHe5ppAGgT8gnrC3lt3ZuQ=";
+    "openvino_detokenizer.xml" = "sha256-bhBqFPFLB3G0a3lIqZsdgZ/5OyRVt9qPR3Yaudup3FY=";
+    "openvino_encoder_model.bin" = "sha256-BZCo81+W1XgBxVmQAo2ReCGschAm40t/P1nXVh/JCOY=";
+    "openvino_encoder_model.xml" = "sha256-YHE9TtOorI7gIOEcRzfsJ20UyrxqCCU3vd8sALps4HA=";
+    "openvino_tokenizer.bin" = "sha256-rfo9mikg0PMUEhJwqWCrMx7A8Fg4VEu47KqkIigqb9Q=";
+    "openvino_tokenizer.xml" = "sha256-y6ME57rVR3O50svM+8hQERfs8uPA9PUzF0Kgo8n+7ZM=";
+    "preprocessor_config.json" = "sha256-ZUzxjT4WO5SM6vl2baVs4LUt4mXVhnPPYck3bxJr1Jk=";
+    "special_tokens_map.json" = "sha256-uupOoJNy60/KhrTkNGE5/XPLgH1Qh+neCUjpcXOcPnQ=";
+    "tokenizer_config.json" = "sha256-PHWUDfzjopT8pwQaX6/wEWd/G2j6heR1EbuM9tzK3tY=";
+    "tokenizer.json" = "sha256-XBvzDJ5xbhR3vt74RrAb4AE9rsuJ6ePveribI8F43xs=";
+    "vocab.json" = "sha256-Z4jICwgumw0TkxR9Oj5iuhkoWsDIKs6OXvAPN+rViXE=";
+  };
+
+  whisperModel = pkgs.runCommandLocal "whisper-large-v3-turbo-int8-ov" { } ''
+    mkdir -p $out
+    ${lib.concatStringsSep "\n" (
+      lib.mapAttrsToList (name: hash: ''
+        ln -s ${
+          pkgs.fetchurl {
+            url = "https://huggingface.co/OpenVINO/whisper-large-v3-turbo-int8-ov/resolve/main/${name}";
+            inherit hash;
+          }
+        } $out/${name}
+      '') modelFiles
+    )}
+  '';
+
+  # openvino-genai 2026.4.0.0 fails to compile against the OpenVINO headers in
+  # nixos-unstable (no matching constructor in continuous_batching); the branch's
+  # nixpkgs builds it, and the service has to match the NPU stack anyway.
+  pythonEnv = pkgs-npu.python3.withPackages (ps: [
+    ps.fastapi
+    ps.numpy
+    ps.openvino-genai
+    # FastAPI's form/file parsing is not part of fastapi itself.
+    ps.python-multipart
+    ps.uvicorn
+  ]);
+in
+{
+  # vinput on the desktop sends raw PCM here and gets text back; the endpoint is
+  # the OpenAI transcription API, so any client of that shape works.
+  systemd.services.whisper-asr = {
+    description = "OpenAI-compatible Whisper transcription service (Intel NPU)";
+    wantedBy = [ "multi-user.target" ];
+    after = [ "network.target" ];
+
+    environment = {
+      WHISPER_MODEL_DIR = "${whisperModel}";
+      WHISPER_MODEL_NAME = "whisper-large-v3-turbo";
+      WHISPER_DEVICE = "NPU";
+      WHISPER_HOST = ipv4;
+      WHISPER_PORT = toString port;
+      # Compiling the graph for the NPU takes ~70 s; cache it so restarts and
+      # reboots reuse the compiled blob instead of paying that again.
+      OV_CACHE_DIR = "/var/cache/whisper-asr";
+      # Python block-buffers stdout when it is not a tty, which would hold the
+      # request log lines back instead of putting them in the journal.
+      PYTHONUNBUFFERED = "1";
+      # The stock driver cannot find the NPU compiler; the patched one loads it
+      # from its own store path.
+      ZE_ENABLE_ALT_DRIVERS = "${lib.getLib pkgs-npu.intel-npu-driver}/lib/libze_intel_npu.so";
+    };
+
+    serviceConfig = {
+      ExecStart = "${pythonEnv}/bin/python ${./asr/server.py}";
+      CacheDirectory = "whisper-asr";
+      DynamicUser = true;
+      Restart = "on-failure";
+      RestartSec = 5;
+
+      # The service only reads its model, writes its cache and listens on the
+      # LAN; it needs no other part of the filesystem or any privilege.
+      NoNewPrivileges = true;
+      PrivateTmp = true;
+      ProtectHome = true;
+      ProtectSystem = "strict";
+      RestrictAddressFamilies = [
+        "AF_INET"
+        "AF_UNIX"
+      ];
+    };
+  };
+}

hosts/idols-aquamarine/asr/server.py

@@ -0,0 +1,200 @@
+"""OpenAI-compatible speech-to-text served by OpenVINO GenAI on the Intel NPU.
+
+vinput records raw PCM (S16_LE, mono, 16 kHz), wraps it in a WAV container and
+POSTs it to /v1/audio/transcriptions; this service is that endpoint. Whisper is
+the only ASR family OpenVINO publishes fully static IRs for, and a static graph
+is what the NPU compiler accepts -- every other family keeps a dynamic time axis
+and aborts the compiler, see WORKAROUNDS.md WA-037.
+"""
+
+import io
+import logging
+import os
+import threading
+import wave
+from contextlib import asynccontextmanager
+
+import numpy as np
+import openvino_genai
+import uvicorn
+from fastapi import FastAPI, File, Form, HTTPException, UploadFile
+from fastapi.concurrency import run_in_threadpool
+from fastapi.responses import JSONResponse, PlainTextResponse
+
+MODEL_DIR = os.environ["WHISPER_MODEL_DIR"]
+MODEL_NAME = os.environ.get("WHISPER_MODEL_NAME", "whisper-large-v3-turbo")
+DEVICE = os.environ.get("WHISPER_DEVICE", "NPU")
+# Empty auto-detects the language per request; set it to pin one language.
+DEFAULT_LANGUAGE = os.environ.get("WHISPER_LANGUAGE") or ""
+CACHE_DIR = os.environ.get("OV_CACHE_DIR", "")
+LISTEN_HOST = os.environ.get("WHISPER_HOST", "127.0.0.1")
+LISTEN_PORT = int(os.environ.get("WHISPER_PORT", "18080"))
+SAMPLE_RATE = 16000
+MAX_AUDIO_SECONDS = int(os.environ.get("WHISPER_MAX_AUDIO_SECONDS") or "120")
+
+logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
+log = logging.getLogger("whisper-asr")
+
+# The idle pipeline, used by the next request. An auto-detected request leaves
+# it dirty and prewarm() replaces it, so at most one instance is alive at rest.
+pipeline = None
+loaded = False
+load_error = None
+# One NPU: compile and infer one request at a time instead of contending for it.
+infer_lock = threading.Lock()
+slot_lock = threading.Lock()
+build_lock = threading.Lock()
+
+
+def build_pipeline() -> openvino_genai.WhisperPipeline:
+    properties = {"CACHE_DIR": CACHE_DIR} if CACHE_DIR else {}
+    return openvino_genai.WhisperPipeline(MODEL_DIR, DEVICE, **properties)
+
+
+def prewarm() -> None:
+    """Build the idle pipeline the next auto-detected request will use.
+
+    Takes ~2 s with a warm NPU cache and runs while the user is not talking."""
+    global pipeline
+    if not build_lock.acquire(blocking=False):
+        return
+    try:
+        with slot_lock:
+            if pipeline is not None:
+                return
+        with infer_lock:
+            fresh = build_pipeline()
+        with slot_lock:
+            if pipeline is None:
+                pipeline = fresh
+    except Exception:  # the next request falls back to building it inline
+        log.exception("could not prewarm a pipeline")
+    finally:
+        build_lock.release()
+
+
+def take_pipeline() -> openvino_genai.WhisperPipeline | None:
+    global pipeline
+    with slot_lock:
+        idle, pipeline = pipeline, None
+    return idle
+
+
+def put_pipeline(idle: openvino_genai.WhisperPipeline) -> None:
+    global pipeline
+    with slot_lock:
+        pipeline = idle
+
+
+def load_pipeline() -> None:
+    """Compile the model. The NPU takes ~70 s on a cold cache, so this runs in
+    the background and /healthz reports readiness instead of blocking startup."""
+    global pipeline, loaded, load_error
+    log.info("loading %s on %s", MODEL_DIR, DEVICE)
+    try:
+        fresh = build_pipeline()
+    except Exception as exc:  # reported through /healthz, not just the journal
+        load_error = f"{type(exc).__name__}: {exc}"
+        log.exception("model failed to load")
+        return
+    with slot_lock:
+        pipeline = fresh
+    loaded = True
+    log.info("model loaded")
+
+
+@asynccontextmanager
+async def lifespan(_app: FastAPI):
+    threading.Thread(target=load_pipeline, daemon=True).start()
+    yield
+
+
+app = FastAPI(lifespan=lifespan)
+
+
+def pcm_from_wav(data: bytes) -> np.ndarray:
+    with wave.open(io.BytesIO(data)) as wav:
+        channels, width, rate = wav.getnchannels(), wav.getsampwidth(), wav.getframerate()
+        if (channels, width, rate) != (1, 2, SAMPLE_RATE):
+            raise ValueError(
+                f"expected mono 16-bit 16 kHz, got {channels}ch/{width * 8}bit/{rate}Hz"
+            )
+        frames = wav.readframes(wav.getnframes())
+    return np.frombuffer(frames, dtype=np.int16).astype(np.float32) / 32768.0
+
+
+def transcribe(samples: np.ndarray, language: str) -> str:
+    language = language or DEFAULT_LANGUAGE
+    with infer_lock:
+        idle = take_pipeline()
+        if idle is None:
+            # A previous request took it and the rebuild has not finished yet.
+            idle = build_pipeline()
+        if language:
+            # Start from the pipeline's config: a fresh WhisperGenerationConfig()
+            # has none of the tokenizer ids and raises "vector::reserve" inside
+            # generate(); an unset task translates everything into English.
+            config = idle.get_generation_config()
+            config.task = "transcribe"
+            config.language = f"<|{language}|>"
+            text = idle.generate(samples, config).texts[0].strip()
+            # A pinned language never relies on detection, so the pipeline is
+            # still good for the next request.
+            put_pipeline(idle)
+            return text
+        # On the NPU the language detected for one request carries over to the
+        # next, so a pipeline that has already transcribed answers a request in
+        # another language with a translation. This one is stale now.
+        text = idle.generate(samples).texts[0].strip()
+    # Building a pipeline keeps the GIL for ~2 s and would delay the response
+    # that is already computed, so let it leave first and rebuild after.
+    timer = threading.Timer(0.5, prewarm)
+    timer.daemon = True
+    timer.start()
+    return text
+
+
+@app.get("/healthz")
+def healthz():
+    if loaded:
+        return {"status": "ok"}
+    if load_error is not None:
+        return JSONResponse({"status": "error", "detail": load_error}, status_code=503)
+    return {"status": "loading"}
+
+
+@app.get("/v1/models")
+def list_models():
+    return {"object": "list", "data": [{"id": MODEL_NAME, "object": "model", "owned_by": "local"}]}
+
+
+@app.post("/v1/audio/transcriptions")
+async def transcriptions(
+    file: UploadFile = File(...),
+    model: str = Form(""),
+    language: str = Form(""),
+    response_format: str = Form("json"),
+):
+    if not loaded:
+        raise HTTPException(status_code=503, detail=load_error or "model is still loading")
+    try:
+        samples = pcm_from_wav(await file.read())
+    except (wave.Error, ValueError, EOFError) as exc:
+        raise HTTPException(status_code=400, detail=f"unsupported audio: {exc}") from exc
+    if not samples.size:
+        raise HTTPException(status_code=400, detail="empty audio")
+    if samples.size > MAX_AUDIO_SECONDS * SAMPLE_RATE:
+        raise HTTPException(status_code=400, detail=f"audio longer than {MAX_AUDIO_SECONDS} s")
+    log.info("transcribing %.1f s of audio", samples.size / SAMPLE_RATE)
+    try:
+        text = await run_in_threadpool(transcribe, samples, language)
+    except Exception as exc:
+        log.exception("transcription failed")
+        raise HTTPException(status_code=500, detail=f"transcription failed: {exc}") from exc
+    if response_format == "text":
+        return PlainTextResponse(text)
+    return {"text": text}
+
+
+if __name__ == "__main__":
+    uvicorn.run(app, host=LISTEN_HOST, port=LISTEN_PORT, log_level="info")

hosts/idols-aquamarine/default.nix

@@ -2,6 +2,7 @@
   disko,
   myvars,
   lib,
+  pkgs-npu,
   ...
 }:
 #############################################################
@@ -30,6 +31,9 @@ in
 
     ./preservation.nix
     ./secureboot.nix
+
+    # services
+    ./asr.nix
   ];
 
   # Bind the board's NCT6798D SuperIO, otherwise fan RPM and board temperatures
@@ -68,6 +72,13 @@ in
   # so enabled user services can run without an active login session.
   users.users."${myvars.username}".linger = true;
 
+  # Speech recognition runs on the NPU, see ./asr.nix. The driver and the
+  # compiler it loads come from NixOS/nixpkgs#570230, which is unmerged, so they
+  # are taken from that branch as `pkgs-npu`; the stock driver cannot find the
+  # compiler, the patched one loads it from its own store path. The service sets
+  # this itself -- the session variable is for manual runs.
+  environment.sessionVariables.ZE_ENABLE_ALT_DRIVERS = "${lib.getLib pkgs-npu.intel-npu-driver}/lib/libze_intel_npu.so";
+
   # The daemons of the hosts that build here (modules/nixos/base/remote-builders.nix).
   users.users.root.openssh.authorizedKeys.keys = myvars.remoteBuilderClientKeys;