From 9f092808fde3b7049e5b356df36c81e9bf9642b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?ryan4yin=20=7C=20=E4=BA=8C=E8=8A=B1?= Date: Sun, 18 Oct 2026 08:34:50 +0800 Subject: [PATCH] feat(aquamarine): serve Whisper transcription from the NPU (#472) The goal is speech recognition on the NPU for vinput, and sherpa-onnx cannot deliver it: every model it loads keeps a dynamic time axis, which the NPU compiler rejects outright. Qwen3-ASR and SenseVoice both core-dumped in vpux-compiler ("Got non broadcastable dimensions pair"), and reshaping the ONNX inputs does not help because the dynamic dimensions sit inside the graph (WA-037). Whisper is the only family OpenVINO publishes fully static IRs for, and its fixed 30 s window is exactly what the compiler accepts. The host now runs an OpenAI-compatible /v1/audio/transcriptions service backed by whisper-large-v3-turbo int8 on the NPU: 1.5 s for 11 s of audio against 2.1 s on CPU, with the compiled graph cached in /var/cache so only a cold cache pays the ~60 s NPU compile. vinput on the desktop sends raw PCM to it through a small provider script. Only the NPU driver and compiler still come from the unmerged branch in the `nixpkgs-npu` input (NixOS/nixpkgs#570230), whose nixpkgs also happens to be the one where openvino-genai 2026.4.0.0 builds at all. Assisted-by: DeepSeek Harness 0.2.1-rc.2 (deepseek-v4.1-flash) --- WORKAROUNDS.md | 25 +-- flake.lock | 17 ++ flake.nix | 4 + home/linux/gui/base/vinput/aqua-provider.py | 78 ++++++++ home/linux/gui/base/vinput/config.json | 11 +- home/linux/gui/base/vinput/default.nix | 6 + hosts/idols-aquamarine/asr.nix | 108 +++++++++++ hosts/idols-aquamarine/asr/server.py | 200 ++++++++++++++++++++ hosts/idols-aquamarine/default.nix | 11 ++ outputs/default.nix | 4 + 10 files changed, 451 insertions(+), 13 deletions(-) create mode 100644 home/linux/gui/base/vinput/aqua-provider.py create mode 100644 hosts/idols-aquamarine/asr.nix create mode 100644 hosts/idols-aquamarine/asr/server.py diff --git a/WORKAROUNDS.md b/WORKAROUNDS.md index cffc6378..0f88c755 100644 --- a/WORKAROUNDS.md +++ b/WORKAROUNDS.md @@ -16,18 +16,19 @@ obsolete workarounds get rolled back instead of lingering. A version or revision pinned for a reason that can go away. -| ID | What & where | Why | Removal condition | Added | Revisit | Status | -| ------ | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------- | ---------- | -------------------------- | ------ | -| WA-001 | smartctl_exporter pinned to upstream `6274decc`, which carries PR [#329](https://github.com/prometheus-community/smartctl_exporter/pull/329) — `overlays/smartctl-exporter/` | `/metrics` returns HTTP 500 once a device becomes readable after startup ([#305](https://github.com/prometheus-community/smartctl_exporter/issues/305), [#326](https://github.com/prometheus-community/smartctl_exporter/issues/326)) | Drop once nixpkgs provides a version containing the #329 fix | 2026-10-03 | next nixpkgs bump | active | -| WA-002 | `nixpkgs-2505` — `flake.nix`; consumer `home/base/tui/container.nix` | `kubernetes-helm` regressed in newer nixpkgs | Drop the input and its consumer once helm works again in current nixpkgs | 2025-12-03 | when helm is revisited | active | -| WA-003 | `rust-overlay` follow — `flake.nix` | lanzaboote v1.1.0's pinned rust-overlay uses the deprecated `stdenv.isLinux`/`stdenv.isDarwin` | Drop the follow once lanzaboote updates its pin (or we drop the lanzaboote pin) | 2026-09-25 | on lanzaboote bump | active | -| WA-004 | `catppuccin` on `main` — `flake.nix` | `main` carries the nodejs→nodejs-slim deprecation fix, which is not in a release | Switch to a release tag once it includes the fix | 2026-09-25 | on catppuccin release | active | -| WA-005 | `tuios` input — `flake.nix`; `home/base/tui/tuios/` | nixpkgs lags behind upstream | Use nixpkgs' `tuios` once it is recent enough | 2026-10-03 | on nixpkgs bump | active | -| WA-006 | `nixpkgs-blender` — `flake.nix` | Blender 5.2 LTS + CUDA/OptiX pin; keeps `just up` from triggering a full source rebuild | Bump deliberately when needed; drop the pin if the current nixpkgs build is acceptable | 2026-07-24 | deliberate | active | -| WA-007 | WeChat from `nixpkgs-master` — `hardening/bwraps/default.nix` | newest builds; the pinned channel lags | Fall back to the stable channel once it has a good build | 2026-10-03 | on nixpkgs bump | active | -| WA-020 | `fcitx5-vinput` pinned to release tag `v2.4.0` — `flake.nix`; consumer `home/linux/gui/base/vinput/default.nix` | Tracks a release tag instead of a branch, so `just up` stays on that release | Bump to a newer release tag when its changes are wanted | 2026-10-04 | on vinput release | active | -| WA-021 | `nix-x86-on-aarch64` input — our fork of rowanG077's patched FEX + muvm runtime — `flake.nix`; `hosts/12kingdoms-shoukei/game/` | FEX does not support 16K pages, so x86/x86_64 apps and games on shoukei need this pinned flake's 4K-page microVM runtime; our fork lets upstream changes be reviewed before use | Drop once FEX supports 16K pages directly, or nixpkgs/nixos-apple-silicon ships an equivalent runtime | 2026-10-07 | on FEX/nixpkgs/muvm update | active | -| WA-023 | `nix.package = pkgs.nixVersions.latest` — `modules/nixos/base/nix.nix` | NixOS is opted into Nix 2.35 (2.35.2), which nixpkgs ships as `nixVersions.latest`, while `nixVersions.stable` is still 2.34 | Drop the override and fall back to the default `pkgs.nix` once `nixVersions.stable` is 2.35 or newer | 2026-10-07 | next nixpkgs bump | active | +| ID | What & where | Why | Removal condition | Added | Revisit | Status | +| ------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------- | ---------- | -------------------------- | ------ | +| WA-001 | smartctl_exporter pinned to upstream `6274decc`, which carries PR [#329](https://github.com/prometheus-community/smartctl_exporter/pull/329) — `overlays/smartctl-exporter/` | `/metrics` returns HTTP 500 once a device becomes readable after startup ([#305](https://github.com/prometheus-community/smartctl_exporter/issues/305), [#326](https://github.com/prometheus-community/smartctl_exporter/issues/326)) | Drop once nixpkgs provides a version containing the #329 fix | 2026-10-03 | next nixpkgs bump | active | +| WA-002 | `nixpkgs-2505` — `flake.nix`; consumer `home/base/tui/container.nix` | `kubernetes-helm` regressed in newer nixpkgs | Drop the input and its consumer once helm works again in current nixpkgs | 2025-12-03 | when helm is revisited | active | +| WA-003 | `rust-overlay` follow — `flake.nix` | lanzaboote v1.1.0's pinned rust-overlay uses the deprecated `stdenv.isLinux`/`stdenv.isDarwin` | Drop the follow once lanzaboote updates its pin (or we drop the lanzaboote pin) | 2026-09-25 | on lanzaboote bump | active | +| WA-004 | `catppuccin` on `main` — `flake.nix` | `main` carries the nodejs→nodejs-slim deprecation fix, which is not in a release | Switch to a release tag once it includes the fix | 2026-09-25 | on catppuccin release | active | +| WA-005 | `tuios` input — `flake.nix`; `home/base/tui/tuios/` | nixpkgs lags behind upstream | Use nixpkgs' `tuios` once it is recent enough | 2026-10-03 | on nixpkgs bump | active | +| WA-006 | `nixpkgs-blender` — `flake.nix` | Blender 5.2 LTS + CUDA/OptiX pin; keeps `just up` from triggering a full source rebuild | Bump deliberately when needed; drop the pin if the current nixpkgs build is acceptable | 2026-07-24 | deliberate | active | +| WA-007 | WeChat from `nixpkgs-master` — `hardening/bwraps/default.nix` | newest builds; the pinned channel lags | Fall back to the stable channel once it has a good build | 2026-10-03 | on nixpkgs bump | active | +| WA-020 | `fcitx5-vinput` pinned to release tag `v2.4.0` — `flake.nix`; consumer `home/linux/gui/base/vinput/default.nix` | Tracks a release tag instead of a branch, so `just up` stays on that release | Bump to a newer release tag when its changes are wanted | 2026-10-04 | on vinput release | active | +| WA-021 | `nix-x86-on-aarch64` input — our fork of rowanG077's patched FEX + muvm runtime — `flake.nix`; `hosts/12kingdoms-shoukei/game/` | FEX does not support 16K pages, so x86/x86_64 apps and games on shoukei need this pinned flake's 4K-page microVM runtime; our fork lets upstream changes be reviewed before use | Drop once FEX supports 16K pages directly, or nixpkgs/nixos-apple-silicon ships an equivalent runtime | 2026-10-07 | on FEX/nixpkgs/muvm update | active | +| WA-023 | `nix.package = pkgs.nixVersions.latest` — `modules/nixos/base/nix.nix` | NixOS is opted into Nix 2.35 (2.35.2), which nixpkgs ships as `nixVersions.latest`, while `nixVersions.stable` is still 2.34 | Drop the override and fall back to the default `pkgs.nix` once `nixVersions.stable` is 2.35 or newer | 2026-10-07 | next nixpkgs bump | active | +| WA-037 | `nixpkgs-npu` input pinned to `2ddccb854d89` — the `openvino-npu-compiler` branch of `ryan4yin/nixpkgs`, carrying [NixOS/nixpkgs#570230](https://github.com/NixOS/nixpkgs/pull/570230) — `flake.nix`; consumer `pkgs-npu` in `outputs/default.nix`, `hosts/idols-aquamarine/asr.nix` | The Intel NPU compiler, and the driver change that loads it, are unmerged, so the NPU driver the Whisper service needs comes from that branch | Drop the input and load `intel-npu-driver` from `nixos-unstable` once #570230 merges | 2026-10-17 | when #570230 merges | active | ## Workarounds diff --git a/flake.lock b/flake.lock index fb77f48b..9e643552 100644 --- a/flake.lock +++ b/flake.lock @@ -940,6 +940,22 @@ "type": "github" } }, + "nixpkgs-npu": { + "locked": { + "lastModified": 1792250783, + "narHash": "sha256-J/gNk34hn774ccDrvJuSWW9EFLvIvyy3t4HnuTcN4tU=", + "owner": "ryan4yin", + "repo": "nixpkgs", + "rev": "2ddccb854d89eb027e09198368073fdb9cde4d1d", + "type": "github" + }, + "original": { + "owner": "ryan4yin", + "ref": "openvino-npu-compiler", + "repo": "nixpkgs", + "type": "github" + } + }, "nixpkgs-patched": { "locked": { "lastModified": 1791119557, @@ -1148,6 +1164,7 @@ "nixpkgs-blender": "nixpkgs-blender", "nixpkgs-darwin": "nixpkgs-darwin", "nixpkgs-master": "nixpkgs-master", + "nixpkgs-npu": "nixpkgs-npu", "nixpkgs-patched": "nixpkgs-patched", "nixpkgs-stable": "nixpkgs-stable", "nu_scripts": "nu_scripts", diff --git a/flake.nix b/flake.nix index 8fc591f6..adb96ea4 100644 --- a/flake.nix +++ b/flake.nix @@ -38,6 +38,10 @@ nixpkgs-patched.url = "github:ryan4yin/nixpkgs/nixos-unstable-patched"; # get some latest packages from the master branch nixpkgs-master.url = "github:nixos/nixpkgs/master"; + # ryan4yin/nixpkgs branch carrying NixOS/nixpkgs#570230 (the Intel NPU + # compiler and the driver change that loads it). Only aquamarine consumes it, + # through `pkgs-npu`; drop it once the PR merges. + nixpkgs-npu.url = "github:ryan4yin/nixpkgs/openvino-npu-compiler"; # NixOS MicroVMs -- run the VMs directly on the hosts, replacing the former KubeVirt microvm = { url = "github:microvm-nix/microvm.nix"; diff --git a/home/linux/gui/base/vinput/aqua-provider.py b/home/linux/gui/base/vinput/aqua-provider.py new file mode 100644 index 00000000..f009bc26 --- /dev/null +++ b/home/linux/gui/base/vinput/aqua-provider.py @@ -0,0 +1,78 @@ +"""vinput ASR provider: send recorded PCM to the aquamarine Whisper service. + +vinput pipes raw PCM (S16_LE, mono, 16 kHz) to stdin and reads the transcript +from stdout. The service speaks the OpenAI transcription API, so the PCM is +wrapped in a WAV container and POSTed as multipart. +""" + +import io +import json +import os +import sys +import uuid +import wave +from urllib.error import URLError +from urllib.request import Request, urlopen + +URL = os.environ["VINPUT_ASR_URL"] +TIMEOUT = float(os.environ.get("VINPUT_ASR_TIMEOUT", "60")) +SAMPLE_RATE = 16000 +FILE_FIELD = ( + 'Content-Disposition: form-data; name="file"; filename="audio.wav"' +) + + +def as_wav(pcm: bytes) -> bytes: + buffer = io.BytesIO() + with wave.open(buffer, "wb") as wav: + wav.setnchannels(1) + wav.setsampwidth(2) + wav.setframerate(SAMPLE_RATE) + wav.writeframes(pcm) + return buffer.getvalue() + + +def multipart(wav: bytes) -> tuple[bytes, str]: + boundary = f"----vinput{uuid.uuid4().hex}" + body = bytearray() + body += f"--{boundary}\r\n".encode() + body += FILE_FIELD.encode() + b"\r\n" + body += b"Content-Type: audio/wav\r\n\r\n" + body += wav + body += b"\r\n" + body += f"--{boundary}\r\n".encode() + body += b'Content-Disposition: form-data; name="model"\r\n\r\n' + body += b"whisper-large-v3-turbo\r\n" + language = os.environ.get("VINPUT_ASR_LANGUAGE", "") + if language: + body += f"--{boundary}\r\n".encode() + body += b'Content-Disposition: form-data; name="language"\r\n\r\n' + body += f"{language}\r\n".encode() + body += f"--{boundary}--\r\n".encode() + return bytes(body), f"multipart/form-data; boundary={boundary}" + + +def main() -> int: + pcm = sys.stdin.buffer.read() + if not pcm: + print("no audio on stdin", file=sys.stderr) + return 1 + payload, content_type = multipart(as_wav(pcm)) + request = Request( + URL, + data=payload, + headers={"Content-Type": content_type}, + method="POST", + ) + try: + with urlopen(request, timeout=TIMEOUT) as response: + transcript = json.load(response) + except (URLError, OSError, ValueError) as exc: + print(f"transcription request failed: {exc}", file=sys.stderr) + return 1 + print(transcript.get("text", "").strip()) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/home/linux/gui/base/vinput/config.json b/home/linux/gui/base/vinput/config.json index f192b0ed..bbd28738 100644 --- a/home/linux/gui/base/vinput/config.json +++ b/home/linux/gui/base/vinput/config.json @@ -13,7 +13,7 @@ "duck_output_volume": 0.25 }, "asr": { - "active_provider": "sherpa-onnx", + "active_provider": "aqua-whisper", "normalize_audio": true, "input_gain": 1.0, "vad": { @@ -29,6 +29,15 @@ "type": "local", "model": "model.sherpa-onnx.qwen3-asr-0.6b-int8", "timeout_ms": 15000 + }, + { + "id": "aqua-whisper", + "type": "command", + "command": "/home/ryan/.config/vinput/aqua-asr-provider", + "env": { + "VINPUT_ASR_URL": "http://192.168.5.101:18080/v1/audio/transcriptions" + }, + "timeout_ms": 60000 } ] }, diff --git a/home/linux/gui/base/vinput/default.nix b/home/linux/gui/base/vinput/default.nix index 49a6e7fe..94113f8a 100644 --- a/home/linux/gui/base/vinput/default.nix +++ b/home/linux/gui/base/vinput/default.nix @@ -38,6 +38,12 @@ in force = true; }; + # The OpenAI-transcription client vinput runs as a provider. config.json points at this + # absolute path, so the two must stay in sync. + xdg.configFile."vinput/aqua-asr-provider".source = + pkgs.writers.writePython3 "vinput-aqua-asr-provider" { } + ./aqua-provider.py; + systemd.user.services.vinput-daemon = { Unit = { Description = "Fcitx5 Vinput voice input daemon"; diff --git a/hosts/idols-aquamarine/asr.nix b/hosts/idols-aquamarine/asr.nix new file mode 100644 index 00000000..3c2d9e3a --- /dev/null +++ b/hosts/idols-aquamarine/asr.nix @@ -0,0 +1,108 @@ +{ + lib, + myvars, + pkgs, + pkgs-npu, + ... +}: +let + inherit (myvars.networking.hostsAddr.aquamarine) ipv4; + port = 18080; + + # OpenVINO publishes static IRs for Whisper only, and the NPU compiler accepts + # nothing less: any other ASR family keeps a dynamic time axis and aborts it + # (WORKAROUNDS.md WA-037). Pinned by content because the NPU caches the + # compiled graph; a version bump should be deliberate. + modelFiles = { + "added_tokens.json" = "sha256-PFH2bEwh+eEmlwB48RrnenjHSu6N9gbunauobkZxCOA="; + "config.json" = "sha256-v9ksCXVHqxLLQquugAi+WlmpH9xas5rM4kSJ64o+ioY="; + "generation_config.json" = "sha256-Rhf8ykWK87kaEDFDqqyRnBq2aAtVLXq9EIEbcki9d7Q="; + "merges.txt" = "sha256-LfKZCjleNejfvHUR4IwS1WAY2NBGkeATPl1jsh4VTcY="; + "normalizer.json" = "sha256-vxxQfchyTKnPmQNkDaz7adri8A7e5PIc66EGpzkvJt0="; + "openvino_config.json" = "sha256-naOOiM7Aaa1UaZ1ifxxZ02w2o9+1S14btc/fWDLvrwM="; + "openvino_decoder_model.bin" = "sha256-wGSZHLr8Q4FWfSmXK3AT3CQCbenDJtA+sebk/ESqlZ8="; + "openvino_decoder_model.xml" = "sha256-rrCfr78cDL+EuvMPRnY0NgBYIvrzs2U1m43gqgTwMEc="; + "openvino_detokenizer.bin" = "sha256-8rPEeCWhCJUl/2XAyOSSceHe5ppAGgT8gnrC3lt3ZuQ="; + "openvino_detokenizer.xml" = "sha256-bhBqFPFLB3G0a3lIqZsdgZ/5OyRVt9qPR3Yaudup3FY="; + "openvino_encoder_model.bin" = "sha256-BZCo81+W1XgBxVmQAo2ReCGschAm40t/P1nXVh/JCOY="; + "openvino_encoder_model.xml" = "sha256-YHE9TtOorI7gIOEcRzfsJ20UyrxqCCU3vd8sALps4HA="; + "openvino_tokenizer.bin" = "sha256-rfo9mikg0PMUEhJwqWCrMx7A8Fg4VEu47KqkIigqb9Q="; + "openvino_tokenizer.xml" = "sha256-y6ME57rVR3O50svM+8hQERfs8uPA9PUzF0Kgo8n+7ZM="; + "preprocessor_config.json" = "sha256-ZUzxjT4WO5SM6vl2baVs4LUt4mXVhnPPYck3bxJr1Jk="; + "special_tokens_map.json" = "sha256-uupOoJNy60/KhrTkNGE5/XPLgH1Qh+neCUjpcXOcPnQ="; + "tokenizer_config.json" = "sha256-PHWUDfzjopT8pwQaX6/wEWd/G2j6heR1EbuM9tzK3tY="; + "tokenizer.json" = "sha256-XBvzDJ5xbhR3vt74RrAb4AE9rsuJ6ePveribI8F43xs="; + "vocab.json" = "sha256-Z4jICwgumw0TkxR9Oj5iuhkoWsDIKs6OXvAPN+rViXE="; + }; + + whisperModel = pkgs.runCommandLocal "whisper-large-v3-turbo-int8-ov" { } '' + mkdir -p $out + ${lib.concatStringsSep "\n" ( + lib.mapAttrsToList (name: hash: '' + ln -s ${ + pkgs.fetchurl { + url = "https://huggingface.co/OpenVINO/whisper-large-v3-turbo-int8-ov/resolve/main/${name}"; + inherit hash; + } + } $out/${name} + '') modelFiles + )} + ''; + + # openvino-genai 2026.4.0.0 fails to compile against the OpenVINO headers in + # nixos-unstable (no matching constructor in continuous_batching); the branch's + # nixpkgs builds it, and the service has to match the NPU stack anyway. + pythonEnv = pkgs-npu.python3.withPackages (ps: [ + ps.fastapi + ps.numpy + ps.openvino-genai + # FastAPI's form/file parsing is not part of fastapi itself. + ps.python-multipart + ps.uvicorn + ]); +in +{ + # vinput on the desktop sends raw PCM here and gets text back; the endpoint is + # the OpenAI transcription API, so any client of that shape works. + systemd.services.whisper-asr = { + description = "OpenAI-compatible Whisper transcription service (Intel NPU)"; + wantedBy = [ "multi-user.target" ]; + after = [ "network.target" ]; + + environment = { + WHISPER_MODEL_DIR = "${whisperModel}"; + WHISPER_MODEL_NAME = "whisper-large-v3-turbo"; + WHISPER_DEVICE = "NPU"; + WHISPER_HOST = ipv4; + WHISPER_PORT = toString port; + # Compiling the graph for the NPU takes ~70 s; cache it so restarts and + # reboots reuse the compiled blob instead of paying that again. + OV_CACHE_DIR = "/var/cache/whisper-asr"; + # Python block-buffers stdout when it is not a tty, which would hold the + # request log lines back instead of putting them in the journal. + PYTHONUNBUFFERED = "1"; + # The stock driver cannot find the NPU compiler; the patched one loads it + # from its own store path. + ZE_ENABLE_ALT_DRIVERS = "${lib.getLib pkgs-npu.intel-npu-driver}/lib/libze_intel_npu.so"; + }; + + serviceConfig = { + ExecStart = "${pythonEnv}/bin/python ${./asr/server.py}"; + CacheDirectory = "whisper-asr"; + DynamicUser = true; + Restart = "on-failure"; + RestartSec = 5; + + # The service only reads its model, writes its cache and listens on the + # LAN; it needs no other part of the filesystem or any privilege. + NoNewPrivileges = true; + PrivateTmp = true; + ProtectHome = true; + ProtectSystem = "strict"; + RestrictAddressFamilies = [ + "AF_INET" + "AF_UNIX" + ]; + }; + }; +} diff --git a/hosts/idols-aquamarine/asr/server.py b/hosts/idols-aquamarine/asr/server.py new file mode 100644 index 00000000..f6decac1 --- /dev/null +++ b/hosts/idols-aquamarine/asr/server.py @@ -0,0 +1,200 @@ +"""OpenAI-compatible speech-to-text served by OpenVINO GenAI on the Intel NPU. + +vinput records raw PCM (S16_LE, mono, 16 kHz), wraps it in a WAV container and +POSTs it to /v1/audio/transcriptions; this service is that endpoint. Whisper is +the only ASR family OpenVINO publishes fully static IRs for, and a static graph +is what the NPU compiler accepts -- every other family keeps a dynamic time axis +and aborts the compiler, see WORKAROUNDS.md WA-037. +""" + +import io +import logging +import os +import threading +import wave +from contextlib import asynccontextmanager + +import numpy as np +import openvino_genai +import uvicorn +from fastapi import FastAPI, File, Form, HTTPException, UploadFile +from fastapi.concurrency import run_in_threadpool +from fastapi.responses import JSONResponse, PlainTextResponse + +MODEL_DIR = os.environ["WHISPER_MODEL_DIR"] +MODEL_NAME = os.environ.get("WHISPER_MODEL_NAME", "whisper-large-v3-turbo") +DEVICE = os.environ.get("WHISPER_DEVICE", "NPU") +# Empty auto-detects the language per request; set it to pin one language. +DEFAULT_LANGUAGE = os.environ.get("WHISPER_LANGUAGE") or "" +CACHE_DIR = os.environ.get("OV_CACHE_DIR", "") +LISTEN_HOST = os.environ.get("WHISPER_HOST", "127.0.0.1") +LISTEN_PORT = int(os.environ.get("WHISPER_PORT", "18080")) +SAMPLE_RATE = 16000 +MAX_AUDIO_SECONDS = int(os.environ.get("WHISPER_MAX_AUDIO_SECONDS") or "120") + +logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s") +log = logging.getLogger("whisper-asr") + +# The idle pipeline, used by the next request. An auto-detected request leaves +# it dirty and prewarm() replaces it, so at most one instance is alive at rest. +pipeline = None +loaded = False +load_error = None +# One NPU: compile and infer one request at a time instead of contending for it. +infer_lock = threading.Lock() +slot_lock = threading.Lock() +build_lock = threading.Lock() + + +def build_pipeline() -> openvino_genai.WhisperPipeline: + properties = {"CACHE_DIR": CACHE_DIR} if CACHE_DIR else {} + return openvino_genai.WhisperPipeline(MODEL_DIR, DEVICE, **properties) + + +def prewarm() -> None: + """Build the idle pipeline the next auto-detected request will use. + + Takes ~2 s with a warm NPU cache and runs while the user is not talking.""" + global pipeline + if not build_lock.acquire(blocking=False): + return + try: + with slot_lock: + if pipeline is not None: + return + with infer_lock: + fresh = build_pipeline() + with slot_lock: + if pipeline is None: + pipeline = fresh + except Exception: # the next request falls back to building it inline + log.exception("could not prewarm a pipeline") + finally: + build_lock.release() + + +def take_pipeline() -> openvino_genai.WhisperPipeline | None: + global pipeline + with slot_lock: + idle, pipeline = pipeline, None + return idle + + +def put_pipeline(idle: openvino_genai.WhisperPipeline) -> None: + global pipeline + with slot_lock: + pipeline = idle + + +def load_pipeline() -> None: + """Compile the model. The NPU takes ~70 s on a cold cache, so this runs in + the background and /healthz reports readiness instead of blocking startup.""" + global pipeline, loaded, load_error + log.info("loading %s on %s", MODEL_DIR, DEVICE) + try: + fresh = build_pipeline() + except Exception as exc: # reported through /healthz, not just the journal + load_error = f"{type(exc).__name__}: {exc}" + log.exception("model failed to load") + return + with slot_lock: + pipeline = fresh + loaded = True + log.info("model loaded") + + +@asynccontextmanager +async def lifespan(_app: FastAPI): + threading.Thread(target=load_pipeline, daemon=True).start() + yield + + +app = FastAPI(lifespan=lifespan) + + +def pcm_from_wav(data: bytes) -> np.ndarray: + with wave.open(io.BytesIO(data)) as wav: + channels, width, rate = wav.getnchannels(), wav.getsampwidth(), wav.getframerate() + if (channels, width, rate) != (1, 2, SAMPLE_RATE): + raise ValueError( + f"expected mono 16-bit 16 kHz, got {channels}ch/{width * 8}bit/{rate}Hz" + ) + frames = wav.readframes(wav.getnframes()) + return np.frombuffer(frames, dtype=np.int16).astype(np.float32) / 32768.0 + + +def transcribe(samples: np.ndarray, language: str) -> str: + language = language or DEFAULT_LANGUAGE + with infer_lock: + idle = take_pipeline() + if idle is None: + # A previous request took it and the rebuild has not finished yet. + idle = build_pipeline() + if language: + # Start from the pipeline's config: a fresh WhisperGenerationConfig() + # has none of the tokenizer ids and raises "vector::reserve" inside + # generate(); an unset task translates everything into English. + config = idle.get_generation_config() + config.task = "transcribe" + config.language = f"<|{language}|>" + text = idle.generate(samples, config).texts[0].strip() + # A pinned language never relies on detection, so the pipeline is + # still good for the next request. + put_pipeline(idle) + return text + # On the NPU the language detected for one request carries over to the + # next, so a pipeline that has already transcribed answers a request in + # another language with a translation. This one is stale now. + text = idle.generate(samples).texts[0].strip() + # Building a pipeline keeps the GIL for ~2 s and would delay the response + # that is already computed, so let it leave first and rebuild after. + timer = threading.Timer(0.5, prewarm) + timer.daemon = True + timer.start() + return text + + +@app.get("/healthz") +def healthz(): + if loaded: + return {"status": "ok"} + if load_error is not None: + return JSONResponse({"status": "error", "detail": load_error}, status_code=503) + return {"status": "loading"} + + +@app.get("/v1/models") +def list_models(): + return {"object": "list", "data": [{"id": MODEL_NAME, "object": "model", "owned_by": "local"}]} + + +@app.post("/v1/audio/transcriptions") +async def transcriptions( + file: UploadFile = File(...), + model: str = Form(""), + language: str = Form(""), + response_format: str = Form("json"), +): + if not loaded: + raise HTTPException(status_code=503, detail=load_error or "model is still loading") + try: + samples = pcm_from_wav(await file.read()) + except (wave.Error, ValueError, EOFError) as exc: + raise HTTPException(status_code=400, detail=f"unsupported audio: {exc}") from exc + if not samples.size: + raise HTTPException(status_code=400, detail="empty audio") + if samples.size > MAX_AUDIO_SECONDS * SAMPLE_RATE: + raise HTTPException(status_code=400, detail=f"audio longer than {MAX_AUDIO_SECONDS} s") + log.info("transcribing %.1f s of audio", samples.size / SAMPLE_RATE) + try: + text = await run_in_threadpool(transcribe, samples, language) + except Exception as exc: + log.exception("transcription failed") + raise HTTPException(status_code=500, detail=f"transcription failed: {exc}") from exc + if response_format == "text": + return PlainTextResponse(text) + return {"text": text} + + +if __name__ == "__main__": + uvicorn.run(app, host=LISTEN_HOST, port=LISTEN_PORT, log_level="info") diff --git a/hosts/idols-aquamarine/default.nix b/hosts/idols-aquamarine/default.nix index aa635ba3..7d3dba82 100644 --- a/hosts/idols-aquamarine/default.nix +++ b/hosts/idols-aquamarine/default.nix @@ -2,6 +2,7 @@ disko, myvars, lib, + pkgs-npu, ... }: ############################################################# @@ -30,6 +31,9 @@ in ./preservation.nix ./secureboot.nix + + # services + ./asr.nix ]; # Bind the board's NCT6798D SuperIO, otherwise fan RPM and board temperatures @@ -68,6 +72,13 @@ in # so enabled user services can run without an active login session. users.users."${myvars.username}".linger = true; + # Speech recognition runs on the NPU, see ./asr.nix. The driver and the + # compiler it loads come from NixOS/nixpkgs#570230, which is unmerged, so they + # are taken from that branch as `pkgs-npu`; the stock driver cannot find the + # compiler, the patched one loads it from its own store path. The service sets + # this itself -- the session variable is for manual runs. + environment.sessionVariables.ZE_ENABLE_ALT_DRIVERS = "${lib.getLib pkgs-npu.intel-npu-driver}/lib/libze_intel_npu.so"; + # The daemons of the hosts that build here (modules/nixos/base/remote-builders.nix). users.users.root.openssh.authorizedKeys.keys = myvars.remoteBuilderClientKeys; diff --git a/outputs/default.nix b/outputs/default.nix index 1c902eb0..5d95859a 100644 --- a/outputs/default.nix +++ b/outputs/default.nix @@ -46,6 +46,10 @@ let # to use chrome, we need to allow the installation of non-free software config.allowUnfree = true; }; + pkgs-npu = import inputs.nixpkgs-npu { + inherit system; + config.allowUnfree = true; + }; pkgs-blender = import inputs.nixpkgs-blender { inherit system; config = lib.optionalAttrs (system == "x86_64-linux") {