diff --git a/docs/trying-it.md b/docs/trying-it.md index 08ca6aa1..ebf2acdd 100644 --- a/docs/trying-it.md +++ b/docs/trying-it.md @@ -186,6 +186,14 @@ a black window or a guest that does not start. The guest says which it got, in t on the console: `GPU (virgl)` or `software (llvmpipe)`. That is the guest's driver, not the host's: the guest cannot tell a GPU from a host that draws in software. +On WSL 2 the command also sets what Mesa and libva need to find the Windows driver, in the process +that runs QEMU, unless you have set it yourself: `GALLIUM_DRIVER=d3d12`, `LD_LIBRARY_PATH` with +`/usr/lib/wsl/lib` first, and `LIBVA_DRIVER_NAME` and `LIBVA_DRIVERS_PATH` for the video driver of +Mesa for it. With them the host's GPU encodes the H.264 that KRDP sends over RDP: the guest's VA-API +lists an encoder, and KRDP uses it. Without the video driver (`/run/opengl-driver/lib/dri/d3d12_drv_video.so`, +which `wsl.useWindowsDriver = true;` brings on NixOS-WSL) the guest encodes on its CPUs, and +`run-kde --doctor` says so. + ## RDP The guest also runs an RDP server, which is the way in when there is no window. The banner has its diff --git a/nix/dev/krdp.nix b/nix/dev/krdp.nix index a7a21300..4d56e021 100644 --- a/nix/dev/krdp.nix +++ b/nix/dev/krdp.nix @@ -15,6 +15,16 @@ let cfg = config.komorebi.krdp; + # KRDP asks for the H.264 Constrained Baseline profile and nothing else. A guest whose VA-API + # drives the host's encoder (virgl, with the QEMU of run-kde.nix) lists Main and High and no + # Constrained Baseline, so ffmpeg's h264_vaapi finds no profile it can use and KRDP encodes on + # the CPUs. Main is in RDP's H.264 mode as well, and the software encoder takes it too. + krdp = pkgs.kdePackages.krdp.overrideAttrs (old: { + postPatch = (old.postPatch or "") + '' + substituteInPlace src/VideoStreamSurface.cpp --replace-fail H264Baseline H264Main + ''; + }); + startKrdp = pkgs.writeShellScript "krdp-start" '' set -eu dir=$HOME/.config/komorebi-vm @@ -33,7 +43,7 @@ let chmod 600 "$dir/rdp.key" fi # --plasma asks KWin for the screen and for input directly, with no portal prompt to click - exec ${pkgs.kdePackages.krdp}/bin/krdpserver --plasma --address 0.0.0.0 --port 3389 \ + exec ${krdp}/bin/krdpserver --plasma --address 0.0.0.0 --port 3389 \ --certificate "$dir/rdp.crt" --certificate-key "$dir/rdp.key" \ --username "$USER" --password "$(cat "$password_file")" ''; @@ -57,7 +67,7 @@ in config = lib.mkIf cfg.enable { environment.systemPackages = [ - pkgs.kdePackages.krdp + krdp # A client, to try the server from inside the guest. FreeRDP decodes H.264 with OpenH264 # when it is built with it, and OpenH264 shows nothing of the video KRDP sends; without it, # FreeRDP decodes with FFmpeg (the VM suite `rdp`, docs/dev/PLATFORM.md). diff --git a/nix/dev/mesa-virgl-fence-leak.patch b/nix/dev/mesa-virgl-fence-leak.patch new file mode 100644 index 00000000..1a02c9d7 --- /dev/null +++ b/nix/dev/mesa-virgl-fence-leak.patch @@ -0,0 +1,56 @@ +mesa: let go of the fence a surface is flushed or processed again over + +vlVaSurfaceFlush() (the upload of an image, the clear of a new surface) and the end of a +video-processing pass (vl_compositor_proc: the scale_vaapi of FFmpeg, the blit of +vaPutImage) flush into surf->pipe_fence, which gets the new fence of the flush and loses +the old one. The only places that let go of it are vaSyncSurface() and the destruction of +the surface, so a surface that is flushed again before a sync, which one of a pool is, +leaves the fence it had behind. + +With virgl a fence owns a file descriptor: the sync_file that the kernel returns for the +out-fence of the execbuffer (VIRTGPU_EXECBUF_FENCE_FD_OUT), closed when the fence is freed. +The guest's KRDP kept one open for every frame it captured until it had 1024 of them, +after which every ioctl failed with EMFILE ("got error from kernel - expect bad rendering +24") and the picture froze. + +Release the fence a flush is about to replace. The newer one covers it: both are fences of +the same context. + +--- a/src/gallium/frontends/va/surface.c 2026-10-28 17:58:18.769410786 -0700 ++++ b/src/gallium/frontends/va/surface.c 2026-10-28 17:58:30.469583700 -0700 +@@ -812,6 +812,14 @@ + void + vlVaSurfaceFlush(vlVaDriver *drv, vlVaSurface *surf) + { ++ /* A flush makes a new fence, which replaces the one in the surface: the one of the last flush ++ * is what vaSyncSurface waits for, and it is only let go of there and when the surface is ++ * destroyed. Replaced without a release, its fence is never freed, and with a driver whose ++ * fences hold a file descriptor (virgl: a sync_file) a surface that is flushed again and ++ * again, as the ones of a pool are, leaves one open a time until there are no more. */ ++ if (surf->pipe_fence) ++ drv->pipe->screen->fence_reference(drv->pipe->screen, &surf->pipe_fence, NULL); ++ + drv->pipe->flush(drv->pipe, &surf->pipe_fence, + drv->has_external_handles ? 0 : PIPE_FLUSH_ASYNC); + } +--- a/src/gallium/auxiliary/vl/vl_compositor_proc.c 2026-10-28 17:58:18.771011962 -0700 ++++ b/src/gallium/auxiliary/vl/vl_compositor_proc.c 2026-10-28 17:58:30.814280800 -0700 +@@ -185,8 +185,17 @@ + struct pipe_picture_desc *picture) + { + struct vl_compositor_proc *proc = (struct vl_compositor_proc *)codec; ++ struct pipe_context *context = proc->b.context; + +- proc->b.context->flush(proc->b.context, picture->out_pipe_fence, picture->flush_flags); ++ /* out_pipe_fence is the pipe_fence of the target surface (vlVaEndPicture, vlVaPutImage), and ++ * the flush makes a new fence for it. The last one is what a sync waits for, so the one it ++ * replaces is let go of here: left, it is never freed, and with virgl its fence is a ++ * sync_file, one more of them open for every frame of a pooled surface that is processed ++ * again before it is synced. */ ++ if (picture->out_pipe_fence && *picture->out_pipe_fence) ++ context->screen->fence_reference(context->screen, picture->out_pipe_fence, NULL); ++ ++ context->flush(context, picture->out_pipe_fence, picture->flush_flags); + + return 0; + } diff --git a/nix/dev/mesa-virgl-rgb-yuv-csc.patch b/nix/dev/mesa-virgl-rgb-yuv-csc.patch new file mode 100644 index 00000000..f7a7289c --- /dev/null +++ b/nix/dev/mesa-virgl-rgb-yuv-csc.patch @@ -0,0 +1,30 @@ +vl/compositor: give the graphics RGB to YUV conversion its matrix + +compositor_proc_process_frame() fills a matrix for each conversion: for RGB to RGB and +YUV to RGB it fills both yuv2rgb and csc_matrix, for RGB to YUV only rgb2yuv. The compute +compositor reads rgb2yuv, but the graphics compositor uploads csc_matrix (set_csc_matrix() +in vl_compositor_gfx.c) to the constants of its fragment shaders, and for RGB to YUV +csc_matrix still held what vl_compositor_init_state() put in it, the YUV to RGB matrix of +BT.709 for NV12. Only radeonsi sets prefer_compute_for_multimedia, so every other driver +takes the graphics path: on virgl scale_vaapi=format=nv12 of an RGB surface (what +KPipeWire and KRDP do with every frame of a screen) gave Y = 1.164 R + 1.793 B - 0.973, +that is 0 for a gray of 0 to 118 and a wrong picture above. + +Fill csc_matrix with the RGB to YUV matrix too, as the other conversions do. + +diff --git a/src/gallium/auxiliary/vl/vl_compositor_proc.c b/src/gallium/auxiliary/vl/vl_compositor_proc.c +index 7de461e..d794038 100644 +--- a/src/gallium/auxiliary/vl/vl_compositor_proc.c ++++ b/src/gallium/auxiliary/vl/vl_compositor_proc.c +@@ -93,6 +93,11 @@ compositor_proc_process_frame(struct pipe_video_codec *codec, + /* RGB to YUV */ + vl_csc_get_rgbyuv_matrix(param->out_matrix_coefficients, src->buffer_format, dst->buffer_format, + param->in_color_range, param->out_color_range, &proc->cstate.rgb2yuv); ++ /* The compute compositor reads rgb2yuv, but the graphics one (the only one that drivers ++ * without prefer_compute_for_multimedia, like virgl, ever use) uploads csc_matrix to its ++ * shader constants, and it still held the YUV to RGB default of vl_compositor_init_state(): ++ * the RGB to YUV fragment shaders then made nonsense of the picture. */ ++ memcpy(&proc->cstate.csc_matrix, &proc->cstate.rgb2yuv, sizeof(vl_csc_matrix)); + } + + vl_csc_get_primaries_matrix(param->in_color_primaries, param->out_color_primaries, diff --git a/nix/dev/qemu-virgl-video.patch b/nix/dev/qemu-virgl-video.patch new file mode 100644 index 00000000..117c494e --- /dev/null +++ b/nix/dev/qemu-virgl-video.patch @@ -0,0 +1,49 @@ +virtio-gpu: let virglrenderer use video encode and decode + +virglrenderer can pass a guest's VA-API video encode and decode to the host's +(virtio_gpu_drv_video in Mesa), but only when asked: with +VIRGL_RENDERER_USE_VIDEO, and with a get_drm_fd callback to say which DRM node to +open the host's video device on. QEMU passes neither, so a guest's vainfo lists +VideoProc and no encoder. VIRGL_RENDERER_USE_VIDEO is only defined with +VIRGL_RENDERER_UNSTABLE_APIS, which the build has to set. + +--- a/hw/display/virtio-gpu-virgl.c 2026-10-28 15:30:25.853615969 -0700 ++++ b/hw/display/virtio-gpu-virgl.c 2026-10-28 15:30:25.853715164 -0700 +@@ -14,6 +14,7 @@ + #include "qemu/osdep.h" + #include "qemu/error-report.h" + #include "qemu/iov.h" ++#include "qemu/drm.h" + #include "trace.h" + #include "hw/virtio/virtio.h" + #include "hw/virtio/virtio-gpu.h" +@@ -1345,6 +1346,18 @@ + return qemu_console_gl_ctx_make_current(g->parent_obj.scanout[scanout_idx].con, qctx); + } + ++#ifdef VIRGL_RENDERER_USE_VIDEO ++/* ++ * virglrenderer opens the host's video encoder and decoder (VA-API) on a DRM ++ * node of its caller's choosing, and disables video when it has none to ask ++ * for. It owns the fd it gets and closes it. ++ */ ++static int virgl_get_drm_fd(void *opaque) ++{ ++ return qemu_drm_rendernode_open(NULL); ++} ++#endif ++ + static struct virgl_renderer_callbacks virtio_gpu_3d_cbs = { + #if VIRGL_VERSION_MAJOR >= 1 + .version = 3, +@@ -1453,6 +1466,10 @@ + #endif + } + #endif ++#ifdef VIRGL_RENDERER_USE_VIDEO ++ virtio_gpu_3d_cbs.get_drm_fd = virgl_get_drm_fd; ++ flags |= VIRGL_RENDERER_USE_VIDEO; ++#endif + #ifdef VIRGL_RENDERER_D3D11_SHARE_TEXTURE + if (qemu_egl_angle_d3d) { + flags |= VIRGL_RENDERER_D3D11_SHARE_TEXTURE; diff --git a/nix/dev/run-kde.nix b/nix/dev/run-kde.nix index fd587d89..78d82ad4 100644 --- a/nix/dev/run-kde.nix +++ b/nix/dev/run-kde.nix @@ -360,6 +360,31 @@ in }; }; + # Mesa's video compositor (vl_compositor) never loads an RGB to YUV matrix on a driver that takes + # its graphics path, which virgl does: the draw multiplies by the YUV to RGB one of its initial + # state, and the NV12 that scale_vaapi makes of the screen, which KRDP encodes, is the wrong + # picture (nix/dev/mesa-virgl-rgb-yuv-csc.patch). `builtins.path` makes the patch a store path of + # its own, so that Mesa is built once for any way of reading this file. There is no binary cache: + # this is a build of Mesa from source. + # + # The same driver leaks a sync_file, an out-fence of the kernel, for every picture it converts or + # uploads: the surface keeps the fence of its last flush in `pipe_fence`, and the next flush of + # it stores a new one over it, so the first is never released, and its file descriptor is never + # closed. A KRDP session ran out of descriptors (1024) after about 950 frames, and the stream + # froze (nix/dev/mesa-virgl-fence-leak.patch). + hardware.graphics.package = pkgs.mesa.overrideAttrs (old: { + patches = (old.patches or [ ]) ++ [ + (builtins.path { + path = ./mesa-virgl-rgb-yuv-csc.patch; + name = "mesa-virgl-rgb-yuv-csc.patch"; + }) + (builtins.path { + path = ./mesa-virgl-fence-leak.patch; + name = "mesa-virgl-fence-leak.patch"; + }) + ]; + }); + # What only the runner, which is the machine of the person who starts the guest, can say. The # ports are forwarded and the display device is added on its command line, and the memory is # sized to what that machine has: the shared memory that virtiofs needs is a backend of the @@ -369,6 +394,38 @@ in # `run-kde` says whether KVM is there, and `--tcg` lets it start without qemu.forceAccel = lib.mkForce false; qemu.enableSharedMemory = false; + # QEMU never asks virglrenderer for the host's video encoder, so the guest's VA-API lists none + # and KRDP encodes H.264 on the guest's CPUs. The patch does (nix/dev/qemu-virgl-video.patch); + # VIRGL_RENDERER_USE_VIDEO is only defined with VIRGL_RENDERER_UNSTABLE_APIS. Only the runner + # has it: there is no binary cache, so it is a QEMU built from source. The virglrenderer under + # it has four patches, in this order, for a host whose encoder is the d3d12 one of WSL 2: + # chroma-format the guest's Mesa and virglrenderer number the chroma format of a codec + # differently: 4:2:0 was read as 4:2:2, and the encoder refused every codec + # video-memory-upload + # the d3d12 driver has no dma-bufs, so the picture to encode goes to the + # surface through memory + # h264-packed-sps Mesa's VA-API takes the level of the stream from a packed SPS, which + # virglrenderer never sent: level 0 crashed the d3d12 encoder + # ubo0-constants a constant buffer 0 bound as a buffer was never read by the shader, so the + # colour matrix of an RGB to YUV conversion was zero + qemu.package = + (pkgs.qemu_kvm.override { + virglrenderer = pkgs.virglrenderer.overrideAttrs (old: { + patches = (old.patches or [ ]) ++ [ + ./virglrenderer-chroma-format.patch + ./virglrenderer-video-memory-upload.patch + ./virglrenderer-h264-packed-sps.patch + ./virglrenderer-ubo0-constants.patch + ]; + }); + }).overrideAttrs + (old: { + patches = (old.patches or [ ]) ++ [ ./qemu-virgl-video.patch ]; + configureFlags = (old.configureFlags or [ ]) ++ [ + "--extra-cflags=-DVIRGL_RENDERER_UNSTABLE_APIS" + ]; + doCheck = false; + }); qemu.options = [ "-object memory-backend-memfd,id=mem0,size=\${RUN_KDE_MEMORY:-8192}M,share=on" "-machine memory-backend=mem0" diff --git a/nix/dev/virglrenderer-chroma-format.patch b/nix/dev/virglrenderer-chroma-format.patch new file mode 100644 index 00000000..62e1abe1 --- /dev/null +++ b/nix/dev/virglrenderer-chroma-format.patch @@ -0,0 +1,54 @@ +virglrenderer: read the chroma format of a video codec in the guest's numbering + +The guest's Mesa (26.2.5) sends a video codec's chroma format as the number of its own +pipe_video_chroma_format, which since the enum moved to u_format.h is NONE, 400, +420, 422, 440, 444 (0 to 5). The enum of virglrenderer, generated by +u_format_table.py, is 400, 420, 422, 444, 440, NONE, so 4:2:0, which is 2, is read as +4:2:2 and VA-API's encoder, which has no 4:2:2, refuses the codec: "format 0x2 not +supported, supported formats: 0x1". The guest's encode then returns no data at all. + +This reads the number as Mesa's. It is for a guest whose Mesa numbers it that way: a +guest whose Mesa still numbers it as virglrenderer does would be read wrongly. + + +diff --git a/src/vrend/vrend_decode.c b/src/vrend/vrend_decode.c +index 7774a25..c6eb8f2 100644 +--- a/src/vrend/vrend_decode.c ++++ b/src/vrend/vrend_decode.c +@@ -1766,6 +1766,26 @@ static int vrend_decode_get_pipe_resource_layout(struct vrend_context *ctx, cons + #ifdef ENABLE_VIDEO + /* video codec related functions */ + ++/* ++ * The chroma format of a codec comes from the guest as the number of its own enum, and the enum ++ * of Mesa since its pipe_video_chroma_format moved to u_format.h is numbered NONE, 400, 420, 422, ++ * 440, 444 (0 to 5), where this one, which u_format_table.py generates, is 400, 420, 422, 444, 440, ++ * NONE. Taken as is, 4:2:0, which is 2, is 4:2:2 here, and the host's encoder, which has no 4:2:2, ++ * refuses the codec ("format 0x2 not supported, supported formats: 0x1"), after which the guest ++ * encodes nothing. ++ */ ++static uint32_t vrend_decode_chroma_format(uint32_t guest_format) ++{ ++ switch (guest_format) { ++ case 1: return PIPE_VIDEO_CHROMA_FORMAT_400; ++ case 2: return PIPE_VIDEO_CHROMA_FORMAT_420; ++ case 3: return PIPE_VIDEO_CHROMA_FORMAT_422; ++ case 4: return PIPE_VIDEO_CHROMA_FORMAT_440; ++ case 5: return PIPE_VIDEO_CHROMA_FORMAT_444; ++ default: return PIPE_VIDEO_CHROMA_FORMAT_NONE; ++ } ++} ++ + static int vrend_decode_create_video_codec(struct vrend_context *ctx, + const uint32_t *buf, + uint32_t length) +@@ -1778,7 +1798,8 @@ static int vrend_decode_create_video_codec(struct vrend_context *ctx, + uint32_t handle = get_buf_entry(buf, VIRGL_CREATE_VIDEO_CODEC_HANDLE); + uint32_t profile = get_buf_entry(buf, VIRGL_CREATE_VIDEO_CODEC_PROFILE); + uint32_t entrypoint = get_buf_entry(buf, VIRGL_CREATE_VIDEO_CODEC_ENTRYPOINT); +- uint32_t chroma_fmt = get_buf_entry(buf, VIRGL_CREATE_VIDEO_CODEC_CHROMA_FMT); ++ uint32_t chroma_fmt = vrend_decode_chroma_format( ++ get_buf_entry(buf, VIRGL_CREATE_VIDEO_CODEC_CHROMA_FMT)); + uint32_t level = get_buf_entry(buf, VIRGL_CREATE_VIDEO_CODEC_LEVEL); + uint32_t width = get_buf_entry(buf, VIRGL_CREATE_VIDEO_CODEC_WIDTH); + uint32_t height = get_buf_entry(buf, VIRGL_CREATE_VIDEO_CODEC_HEIGHT); diff --git a/nix/dev/virglrenderer-h264-packed-sps.patch b/nix/dev/virglrenderer-h264-packed-sps.patch new file mode 100644 index 00000000..c0c37800 --- /dev/null +++ b/nix/dev/virglrenderer-h264-packed-sps.patch @@ -0,0 +1,192 @@ +virglrenderer: send the SPS of an H.264 stream to encode to the host's VA-API + +Mesa's VA-API takes the level (and the profile) of an H.264 stream to encode only from +the packed sequence header: vlVaHandleVAEncSequenceParameterBufferTypeH264() does not +set seq.level_idc, and vlVaHandleVAEncPackedHeaderDataBufferTypeH264() parses it from +the application's SPS. The video protocol carries no SPS and no level, and virglrenderer +sends no packed header, so on a host whose encoder needs the level, the d3d12 driver of +WSL 2, seq.level_idc is 0 and d3d12_video_encoder_convert_level_h264() reaches +UNREACHABLE("Unsupported H264 level"), which segfaults QEMU in vaEndPicture(). + +On an IDR frame, build a minimal SPS (profile, level, order count type, reference +frames, size, cropping) and give it to VA-API as a packed sequence header. The level is +the smallest of Table A-1 that holds the frame size and the macroblock rate. The d3d12 +encoder writes the SPS and PPS it encodes itself: the packed header only has to say what +it needs to know. + + +diff --git a/src/vrend/virgl_video.c b/src/vrend/virgl_video.c +index 8b47837..8b71ebe 100644 +--- a/src/vrend/virgl_video.c ++++ b/src/vrend/virgl_video.c +@@ -1550,6 +1550,161 @@ err: + return err; + } + ++/* ++ * Mesa's VA-API takes the profile and the level of an H.264 stream to encode only from the ++ * packed sequence header (vlVaHandleVAEncPackedHeaderDataBufferTypeH264 parses the SPS of the ++ * application), and not from the sequence parameter. Nothing in the video protocol carries the ++ * SPS or the level of the guest's, and the level of the codec is not one: with no packed header ++ * seq.level_idc is 0, which the d3d12 encoder (WSL 2) does not take, and it crashes in ++ * d3d12_video_encoder_convert_level_h264(). The d3d12 encoder writes the SPS and the PPS it ++ * encodes itself, and the headers of the application only tell it to, so the SPS here only has to ++ * say what the encoder needs: the profile, the level, the order count type, the number of ++ * reference frames, the size and the cropping. ++ */ ++struct h264_bits { ++ uint8_t *buf; ++ unsigned size; /* bytes in buf */ ++ unsigned pos; /* bits written */ ++}; ++ ++static void h264_put_bits(struct h264_bits *w, uint32_t value, unsigned nbits) ++{ ++ while (nbits--) { ++ if (w->pos / 8 < w->size) { ++ if (value >> nbits & 1) ++ w->buf[w->pos / 8] |= 0x80 >> (w->pos % 8); ++ else ++ w->buf[w->pos / 8] &= ~(0x80 >> (w->pos % 8)); ++ } ++ w->pos++; ++ } ++} ++ ++/* Exp-Golomb ue(v) */ ++static void h264_put_ue(struct h264_bits *w, uint32_t value) ++{ ++ uint32_t v = value + 1; ++ unsigned nbits = 0; ++ ++ while (v >> nbits) ++ nbits++; ++ h264_put_bits(w, 0, nbits - 1); ++ h264_put_bits(w, v, nbits); ++} ++ ++/* the smallest level that holds the frame size and the macroblock rate (Table A-1) */ ++static uint32_t h264_level_idc(uint32_t width_mbs, uint32_t height_mbs, ++ uint32_t fps) ++{ ++ static const struct { uint8_t idc; uint32_t max_mbps; uint32_t max_fs; } levels[] = { ++ { 10, 1485, 99 }, { 11, 3000, 396 }, { 12, 6000, 396 }, ++ { 13, 11880, 396 }, { 20, 11880, 396 }, { 21, 19800, 792 }, ++ { 22, 20250, 1620 }, { 30, 40500, 1620 }, { 31, 108000, 3600 }, ++ { 32, 216000, 5120 }, { 40, 245760, 8192 }, { 41, 245760, 8192 }, ++ { 42, 522240, 8704 }, { 50, 589824, 22080 }, { 51, 983040, 36864 }, ++ { 52, 2073600, 36864 }, { 60, 4177920, 139264 }, ++ }; ++ uint32_t mbs = width_mbs * height_mbs; ++ unsigned i; ++ ++ if (!fps) ++ fps = 30; ++ ++ for (i = 0; i < ARRAY_SIZE(levels); i++) { ++ if (mbs <= levels[i].max_fs && (uint64_t)mbs * fps <= levels[i].max_mbps) ++ return levels[i].idc; ++ } ++ ++ return levels[ARRAY_SIZE(levels) - 1].idc; ++} ++ ++static void h264_encode_render_packed_sps( ++ struct virgl_video_codec *codec, ++ const struct virgl_h264_enc_picture_desc *desc) ++{ ++ uint8_t data[64] = { 0 }; ++ struct h264_bits w = { data, sizeof(data), 0 }; ++ uint32_t profile_idc, fps, level_idc, width_mbs, height_mbs, poc_type; ++ const struct virgl_h264_enc_rate_control *rc = &desc->rate_ctrl[0]; ++ VAEncPackedHeaderParameterBuffer hdr; ++ VABufferID bufs[2]; ++ VAStatus va_stat; ++ unsigned nbytes; ++ ++ switch (codec->profile) { ++ case PIPE_VIDEO_PROFILE_MPEG4_AVC_BASELINE: ++ profile_idc = 66; ++ break; ++ case PIPE_VIDEO_PROFILE_MPEG4_AVC_MAIN: ++ profile_idc = 77; ++ break; ++ default: ++ profile_idc = 100; ++ break; ++ } ++ ++ width_mbs = (codec->width + 15) / 16; ++ height_mbs = (codec->height + 15) / 16; ++ fps = rc->frame_rate_den ? rc->frame_rate_num / rc->frame_rate_den : 0; ++ level_idc = h264_level_idc(width_mbs, height_mbs, fps); ++ /* the order count types that the SPS below can say, 0 and 2 */ ++ poc_type = desc->seq.pic_order_cnt_type == 2 ? 2 : 0; ++ ++ h264_put_bits(&w, 0x00000001, 32); /* start code */ ++ h264_put_bits(&w, 0x67, 8); /* nal_ref_idc 3, SPS */ ++ h264_put_bits(&w, profile_idc, 8); ++ h264_put_bits(&w, desc->seq.enc_constraint_set_flags & 0x3f, 6); ++ h264_put_bits(&w, 0, 2); /* reserved_zero_2bits */ ++ h264_put_bits(&w, level_idc, 8); ++ h264_put_ue(&w, 0); /* seq_parameter_set_id */ ++ if (profile_idc == 100) { ++ h264_put_ue(&w, 1); /* chroma_format_idc: 4:2:0 */ ++ h264_put_ue(&w, 0); /* bit_depth_luma_minus8 */ ++ h264_put_ue(&w, 0); /* bit_depth_chroma_minus8 */ ++ h264_put_bits(&w, 0, 1); /* qpprime_y_zero_transform_bypass */ ++ h264_put_bits(&w, 0, 1); /* seq_scaling_matrix_present */ ++ } ++ h264_put_ue(&w, 0); /* log2_max_frame_num_minus4 */ ++ h264_put_ue(&w, poc_type); ++ if (poc_type == 0) ++ h264_put_ue(&w, 0); /* log2_max_pic_order_cnt_lsb_minus4 */ ++ h264_put_ue(&w, codec->max_references); ++ h264_put_bits(&w, 0, 1); /* gaps_in_frame_num_value_allowed */ ++ h264_put_ue(&w, width_mbs - 1); ++ h264_put_ue(&w, height_mbs - 1); ++ h264_put_bits(&w, 1, 1); /* frame_mbs_only_flag */ ++ h264_put_bits(&w, 1, 1); /* direct_8x8_inference_flag */ ++ h264_put_bits(&w, desc->seq.enc_frame_cropping_flag ? 1 : 0, 1); ++ if (desc->seq.enc_frame_cropping_flag) { ++ h264_put_ue(&w, desc->seq.enc_frame_crop_left_offset); ++ h264_put_ue(&w, desc->seq.enc_frame_crop_right_offset); ++ h264_put_ue(&w, desc->seq.enc_frame_crop_top_offset); ++ h264_put_ue(&w, desc->seq.enc_frame_crop_bottom_offset); ++ } ++ h264_put_bits(&w, 0, 1); /* vui_parameters_present_flag */ ++ h264_put_bits(&w, 1, 1); /* rbsp_stop_one_bit */ ++ nbytes = (w.pos + 7) / 8; ++ if (nbytes > sizeof(data)) ++ return; ++ ++ memset(&hdr, 0, sizeof(hdr)); ++ hdr.type = VAEncPackedHeaderSequence; ++ hdr.bit_length = nbytes * 8; ++ hdr.has_emulation_bytes = 0; ++ ++ vaCreateBuffer(va_dpy, codec->va_ctx, VAEncPackedHeaderParameterBufferType, ++ sizeof(hdr), 1, &hdr, &bufs[0]); ++ vaCreateBuffer(va_dpy, codec->va_ctx, VAEncPackedHeaderDataBufferType, ++ nbytes, 1, data, &bufs[1]); ++ ++ va_stat = vaRenderPicture(va_dpy, codec->va_ctx, bufs, 2); ++ if (VA_STATUS_SUCCESS != va_stat) ++ virgl_error("render h264 packed sps failed, err = 0x%x\n", va_stat); ++ ++ vaDestroyBuffer(va_dpy, bufs[0]); ++ vaDestroyBuffer(va_dpy, bufs[1]); ++} ++ + static int h264_encode_render_sequence( + struct virgl_video_codec *codec, + struct virgl_video_buffer *source, +@@ -1591,6 +1746,8 @@ static int h264_encode_render_sequence( + goto error; + } + ++ h264_encode_render_packed_sps(codec, desc); ++ + va_stat = vaRenderPicture(va_dpy, codec->va_ctx, &rc_param_buf, 1); + if (VA_STATUS_SUCCESS != va_stat) { + virgl_error("render h264 rate control param failed, err = 0x%x\n", va_stat); diff --git a/nix/dev/virglrenderer-ubo0-constants.patch b/nix/dev/virglrenderer-ubo0-constants.patch new file mode 100644 index 00000000..d54b5201 --- /dev/null +++ b/nix/dev/virglrenderer-ubo0-constants.patch @@ -0,0 +1,105 @@ +vrend: read constant buffer 0 bound as a buffer into the default uniforms + +A guest binds the constant buffer 0 of a shader either as inline constants +(SET_CONSTANT_BUFFER) or as a buffer resource (SET_UNIFORM_BUFFER). A shader reads its +constant buffer 0 from the default uniform block (fsconst0[] and the like), which vrend +fills only from the inline constants, and vrend binds UBOs only for the buffers from 1: +a buffer bound at index 0 was never seen by the shader, and its constants stayed zero. + +Mesa's vl_compositor binds the colour matrix of an RGB to YUV conversion that way, so +with the virgl driver the conversion (scale_vaapi from RGB to NV12 in FFmpeg, which +KRDP uses) multiplied by a zero matrix and wrote zeros: a solid green picture. + +Remember when the constant buffer 0 is set whether it is a buffer, and at the draw read +it (glGetBufferSubData) into the uniforms with glUniform4uiv. It is read at every draw, +since the guest can write to the buffer without binding it again. The conversion then +writes what Mesa's matrix gives: the luma of a ramp goes from 0 0 0 to 0 124 255. What +is left is Mesa's: the graphics path of vl_compositor_convert_rgb_to_yuv() never loads +an RGB to YUV matrix, and the one that is sent is the YUV to RGB matrix of the +compositor's initial state. + +diff --git a/src/vrend/vrend_renderer.c b/src/vrend/vrend_renderer.c +index 4bf88e0..a5166c4 100644 +--- a/src/vrend/vrend_renderer.c ++++ b/src/vrend/vrend_renderer.c +@@ -735,6 +735,8 @@ struct vrend_sub_context { + + struct vrend_constants consts[PIPE_SHADER_TYPES]; + bool const_dirty[PIPE_SHADER_TYPES]; ++ /* constant buffer 0 was last bound as a buffer resource, not as inline constants */ ++ bool const0_from_buffer[PIPE_SHADER_TYPES]; + + struct pipe_constant_buffer cbs[PIPE_SHADER_TYPES][PIPE_MAX_CONSTANT_BUFFERS]; + uint32_t const_bufs_used_mask[PIPE_SHADER_TYPES]; +@@ -3486,6 +3488,7 @@ void vrend_set_constants(struct vrend_context *ctx, + + consts = &ctx->sub->consts[shader]; + ctx->sub->const_dirty[shader] = true; ++ ctx->sub->const0_from_buffer[shader] = false; + + /* avoid reallocations by only growing the buffer */ + if (consts->num_allocated_consts < num_constant) { +@@ -3517,6 +3520,9 @@ void vrend_set_uniform_buffer(struct vrend_context *ctx, + struct pipe_constant_buffer *cbs = &ctx->sub->cbs[shader][index]; + const uint32_t mask = 1u << index; + ++ if (index == 0) ++ ctx->sub->const0_from_buffer[shader] = res_handle != 0; ++ + if (res_handle) { + res = vrend_renderer_ctx_res_lookup(ctx, res_handle); + +@@ -5352,9 +5358,53 @@ static GLuint vrend_draw_bind_ubo_shader(struct vrend_sub_context *sub_ctx, + return next_ubo_id; + } + ++/* ++ * A guest can bind constant buffer 0 as a buffer resource (SET_UNIFORM_BUFFER) instead of as ++ * inline constants (SET_CONSTANT_BUFFER): Mesa's vl_compositor does for the colour matrix of an ++ * RGB to YUV conversion. A shader reads its constant buffer 0 as the default uniform block ++ * (fsconst0[]), which only the inline constants fill, and the UBOs are for the buffers from 1: ++ * the matrix stayed zero, and the conversion wrote zeros. Read the buffer into the uniforms. It ++ * is read at every draw, as the guest can write to the buffer without binding it again. Returns ++ * whether the constants came from the buffer. ++ */ ++static bool vrend_draw_bind_const0_buffer(struct vrend_sub_context *sub_ctx, ++ int shader_type) ++{ ++ const struct vrend_shader_selector *sel = sub_ctx->shaders[shader_type]; ++ const struct pipe_constant_buffer *cb = &sub_ctx->cbs[shader_type][0]; ++ const struct vrend_resource *res = (const struct vrend_resource *)cb->buffer; ++ unsigned num_consts, bytes; ++ uint32_t *data; ++ ++ if (!sub_ctx->const0_from_buffer[shader_type] || !sel || !sel->sinfo.num_consts || ++ sub_ctx->prog->const_location[shader_type] == -1 || ++ !(sub_ctx->const_bufs_used_mask[shader_type] & 1) || ++ !res || !has_bit(res->storage_bits, VREND_STORAGE_GL_BUFFER)) ++ return false; ++ ++ num_consts = sel->sinfo.num_consts; ++ bytes = num_consts * 4 * sizeof(uint32_t); ++ data = calloc(1, bytes); ++ if (!data) ++ return true; ++ ++ glBindBuffer(GL_COPY_READ_BUFFER, res->gl_id); ++ glGetBufferSubData(GL_COPY_READ_BUFFER, cb->buffer_offset, ++ MIN2(bytes, cb->buffer_size), data); ++ glBindBuffer(GL_COPY_READ_BUFFER, 0); ++ ++ glUniform4uiv(sub_ctx->prog->const_location[shader_type], num_consts, data); ++ free(data); ++ ++ return true; ++} ++ + static void vrend_draw_bind_const_shader(struct vrend_sub_context *sub_ctx, + int shader_type, bool new_program) + { ++ if (vrend_draw_bind_const0_buffer(sub_ctx, shader_type)) ++ return; ++ + if (sub_ctx->consts[shader_type].consts && + sub_ctx->shaders[shader_type] && + (sub_ctx->prog->const_location[shader_type] != -1) && diff --git a/nix/dev/virglrenderer-video-memory-upload.patch b/nix/dev/virglrenderer-video-memory-upload.patch new file mode 100644 index 00000000..bf67e0ed --- /dev/null +++ b/nix/dev/virglrenderer-video-memory-upload.patch @@ -0,0 +1,237 @@ +virglrenderer: upload the picture to encode through memory + +virglrenderer shares a video buffer with the host's VA-API driver as a dma-buf: it +exports the VA surface (vaExportSurfaceHandle) and imports it into EGL to copy the +guest's picture in. The d3d12 driver on WSL 2 has no dma-bufs, so the export fails +("export surface failed, err = 0x6"), nothing is uploaded and the encoder returns an +empty bitstream. + +When the first export fails, read the planes of the guest's picture back from their +textures (glReadPixels, into the planes of a VAImage) and write them to the surface +through vaDeriveImage(), as FFmpeg's hwupload does; where the driver cannot derive an +image, a vaCreateImage() image is copied with vaPutImage(). On the d3d12 driver the +vaPutImage() copy came out with an all-zero luma plane (the chroma was right), where +the derived image encodes exactly: a luma ramp through the encoder comes back as +43 125 207 for 43 125 207. Where the export works nothing changes: it is tried first, +and given up on only after it has failed. + +diff --git a/src/vrend/virgl_video.c b/src/vrend/virgl_video.c +index dc1e8cd..8b47837 100644 +--- a/src/vrend/virgl_video.c ++++ b/src/vrend/virgl_video.c +@@ -438,12 +438,100 @@ static void destroy_video_dma_buf(struct virgl_video_dma_buf *dmabuf) + } + } + ++/* ++ * Upload the picture to encode through memory, for a host that cannot export ++ * a video buffer as a dma-buf: the d3d12 driver on WSL 2 has no dma-bufs, and ++ * vaExportSurfaceHandle() fails on it. The callback reads the planes of the ++ * guest's picture into a VAImage, which vaPutImage() copies to the surface, ++ * as FFmpeg's hwupload does. ++ */ ++static void encode_upload_picture_cpu(struct virgl_video_codec *codec, ++ struct virgl_video_buffer *buffer) ++{ ++ VAStatus va_stat; ++ VAImageFormat fmt; ++ VAImage image; ++ void *data = NULL; ++ uint8_t *dst[3] = { NULL, NULL, NULL }; ++ unsigned pitches[3] = { 0, 0, 0 }; ++ unsigned i; ++ int ret; ++ bool derived = false; ++ ++ if (!callbacks || !callbacks->encode_read_picture) ++ return; ++ ++ memset(&fmt, 0, sizeof(fmt)); ++ fmt.byte_order = VA_LSB_FIRST; ++ fmt.bits_per_pixel = 12; ++ switch (buffer->format) { ++ case PIPE_FORMAT_NV12: ++ fmt.fourcc = VA_FOURCC_NV12; ++ break; ++ case PIPE_FORMAT_NV21: ++ fmt.fourcc = VA_FOURCC_NV21; ++ break; ++ default: ++ virgl_error("upload through memory: pipe format %d not supported\n", ++ buffer->format); ++ return; ++ } ++ ++ /* ++ * Write to the surface itself when the driver can show it as an image, as FFmpeg's ++ * hwupload does. On the d3d12 driver the picture of an image made with vaCreateImage() and ++ * copied by vaPutImage() came out with an all-zero luma plane (the chroma was right), ++ * where the same planes written through vaDeriveImage() encode exactly. ++ */ ++ va_stat = vaDeriveImage(va_dpy, buffer->va_sfc, &image); ++ if (VA_STATUS_SUCCESS == va_stat) { ++ derived = true; ++ } else { ++ va_stat = vaCreateImage(va_dpy, &fmt, buffer->width, buffer->height, &image); ++ if (VA_STATUS_SUCCESS != va_stat) { ++ virgl_error("create image failed, err = 0x%x\n", va_stat); ++ return; ++ } ++ } ++ ++ va_stat = vaMapBuffer(va_dpy, image.buf, &data); ++ if (VA_STATUS_SUCCESS != va_stat || !data) { ++ virgl_error("map image failed, err = 0x%x\n", va_stat); ++ goto destroy_image; ++ } ++ ++ for (i = 0; i < image.num_planes && i < ARRAY_SIZE(dst); i++) { ++ dst[i] = (uint8_t *)data + image.offsets[i]; ++ pitches[i] = image.pitches[i]; ++ } ++ ++ ret = callbacks->encode_read_picture(codec, buffer, image.num_planes, ++ dst, pitches); ++ vaUnmapBuffer(va_dpy, image.buf); ++ if (ret) { ++ virgl_error("read picture failed, err = %d\n", ret); ++ goto destroy_image; ++ } ++ ++ if (!derived) { ++ va_stat = vaPutImage(va_dpy, buffer->va_sfc, image.image_id, ++ 0, 0, buffer->width, buffer->height, ++ 0, 0, buffer->width, buffer->height); ++ if (VA_STATUS_SUCCESS != va_stat) ++ virgl_error("put image failed, err = 0x%x\n", va_stat); ++ } ++ ++destroy_image: ++ vaDestroyImage(va_dpy, image.image_id); ++} ++ + static void encode_upload_picture(struct virgl_video_codec *codec, + struct virgl_video_buffer *buffer) + { + VAStatus va_stat; ++ static bool dmabuf_unsupported; + +- if (!callbacks || !callbacks->encode_upload_picture) ++ if (!callbacks) + return; + + va_stat = vaSyncSurface(va_dpy, buffer->va_sfc); +@@ -452,11 +540,20 @@ static void encode_upload_picture(struct virgl_video_codec *codec, + return; + } + +- if (!buffer->dmabuf) ++ /* once a surface cannot be exported none can: go through memory */ ++ if (!buffer->dmabuf && !dmabuf_unsupported && callbacks->encode_upload_picture) { + buffer->dmabuf = export_video_dma_buf(buffer, VIRGL_VIDEO_DMABUF_WRITE_ONLY); ++ if (!buffer->dmabuf) { ++ dmabuf_unsupported = true; ++ virgl_warn("cannot export a video buffer as a dma-buf, " ++ "uploading the pictures to encode through memory\n"); ++ } ++ } + + if (buffer->dmabuf) + callbacks->encode_upload_picture(codec, buffer->dmabuf); ++ else ++ encode_upload_picture_cpu(codec, buffer); + } + + static void encode_completed(struct virgl_video_codec *codec, +diff --git a/src/vrend/virgl_video.h b/src/vrend/virgl_video.h +index e954559..c39d33d 100644 +--- a/src/vrend/virgl_video.h ++++ b/src/vrend/virgl_video.h +@@ -115,6 +115,16 @@ struct virgl_video_callbacks { + void (*encode_upload_picture)(struct virgl_video_codec *codec, + const struct virgl_video_dma_buf *dmabuf); + ++ /* Read the picture data to be encoded into memory, plane by plane, for ++ * a host whose video buffers cannot be exported as dma-bufs (the d3d12 ++ * driver of WSL 2): plane i is written to dst[i], pitches[i] bytes a row. ++ * Returns 0 when the planes are filled. */ ++ int (*encode_read_picture)(struct virgl_video_codec *codec, ++ struct virgl_video_buffer *buffer, ++ unsigned num_planes, ++ uint8_t * const *dst, ++ const unsigned *pitches); ++ + /* Callback when encoding is complete, used to download the encoded data + * and reference picture */ + void (*encode_completed)(struct virgl_video_codec *codec, +diff --git a/src/vrend/vrend_video.c b/src/vrend/vrend_video.c +index 57341d8..dc30c55 100644 +--- a/src/vrend/vrend_video.c ++++ b/src/vrend/vrend_video.c +@@ -293,6 +293,58 @@ static void vrend_video_enocde_upload_picture( + sync_video_buffer_to_dmabuf(buf, dmabuf); + } + ++/* ++ * The picture to encode is the planes of a video buffer the guest rendered to, which are ++ * textures here (R8 for the luma, RG8 for the chroma of NV12). Read them into the planes of a ++ * VAImage, for a host that cannot share its video buffer as a dma-buf. ++ */ ++static int vrend_video_encode_read_picture( ++ struct virgl_video_codec *codec, ++ struct virgl_video_buffer *vbuf, ++ unsigned num_planes, ++ uint8_t * const *dst, ++ const unsigned *pitches) ++{ ++ struct vrend_video_buffer *buf = vrend_video_buffer(vbuf); ++ int ret = 0; ++ ++ (void)codec; ++ ++ if (!buf) ++ return -1; ++ ++ glBindBuffer(GL_PIXEL_PACK_BUFFER, 0); ++ glPixelStorei(GL_PACK_ALIGNMENT, 1); ++ ++ for (unsigned i = 0; i < num_planes && i < buf->num_planes; i++) { ++ struct vrend_video_plane *plane = &buf->planes[i]; ++ struct vrend_resource *res; ++ /* NV12 and NV21 have a plane of one byte a pixel, and one of two */ ++ const unsigned bytes_per_pixel = i ? 2 : 1; ++ ++ res = vrend_renderer_ctx_res_lookup(buf->ctx->ctx, plane->res_handle); ++ if (!res) { ++ virgl_error("%s: res %d not found\n", __func__, plane->res_handle); ++ ret = -1; ++ continue; ++ } ++ ++ glBindFramebuffer(GL_READ_FRAMEBUFFER, plane->framebuffer); ++ glFramebufferTexture2D(GL_READ_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, ++ GL_TEXTURE_2D, res->gl_id, 0); ++ ++ glPixelStorei(GL_PACK_ROW_LENGTH, pitches[i] / bytes_per_pixel); ++ glReadPixels(0, 0, res->base.width0, res->base.height0, ++ i ? GL_RG : GL_RED, GL_UNSIGNED_BYTE, dst[i]); ++ } ++ ++ glPixelStorei(GL_PACK_ROW_LENGTH, 0); ++ glPixelStorei(GL_PACK_ALIGNMENT, 4); ++ glBindFramebuffer(GL_FRAMEBUFFER, 0); ++ ++ return ret; ++} ++ + static void vrend_video_encode_completed( + struct virgl_video_codec *codec, + const struct virgl_video_dma_buf *src_buf, +@@ -349,6 +401,7 @@ static void vrend_video_encode_completed( + static struct virgl_video_callbacks video_callbacks = { + .decode_completed = vrend_video_decode_completed, + .encode_upload_picture = vrend_video_enocde_upload_picture, ++ .encode_read_picture = vrend_video_encode_read_picture, + .encode_completed = vrend_video_encode_completed, + }; + diff --git a/nix/tests/run-kde.nix b/nix/tests/run-kde.nix index 47fc64d9..80314f4b 100644 --- a/nix/tests/run-kde.nix +++ b/nix/tests/run-kde.nix @@ -15,8 +15,8 @@ let # not keep. setup = '' t=$PWD/machine - mkdir -p $t/bin $t/repo $t/dri - touch $t/repo/flake.nix $t/kvm $t/kvm-denied $t/fuse $t/dxg + mkdir -p $t/bin $t/repo $t/dri $t/wsl-lib $t/va + touch $t/repo/flake.nix $t/kvm $t/kvm-denied $t/fuse $t/dxg $t/va/d3d12_drv_video.so chmod 000 $t/kvm-denied printf '#!/bin/sh\n' > $t/bin/nix printf '#!/bin/sh\necho "[ERROR virtiofsd] Error creating the vhost-user socket: Address in use (os error 98)" >&2\nexit 1\n' > $t/bin/virtiofsd-broken @@ -34,6 +34,7 @@ let RUN_KDE_RUNNER_BIN=/nix/store/fake-nixos-vm/bin/run-komorebi-demo-vm \ RUN_KDE_CPUINFO=$t/cpuinfo-svm RUN_KDE_STORE_DIR=$t RUN_KDE_FREE_GB=100 \ RUN_KDE_DRI_DIR=$t/dri RUN_KDE_CPUS=12 RUN_KDE_FUSE_DEVICE=$t/fuse \ + RUN_KDE_WSL_LIB_DIR=$t/wsl-lib RUN_KDE_VA_DRIVERS_DIR=$t/va \ "$@" } plain() { sed "s|$t|@T@|g"; } @@ -72,6 +73,14 @@ in (scenario "dry-run-wslg-gpu" 0 ''machine ${wslg} RUN_KDE_DXG_DEVICE=$t/dxg "RUN_KDE_HOST_RENDERER=D3D12 (AMD Radeon RX 7800 XT)" ${exe} --dry-run'' ) + # without the d3d12 video driver the host's GPU still renders, and nothing says libva which driver + (scenario "dry-run-wslg-gpu-no-video-driver" 0 + ''machine ${wslg} RUN_KDE_DXG_DEVICE=$t/dxg "RUN_KDE_HOST_RENDERER=D3D12 (AMD Radeon RX 7800 XT)" RUN_KDE_VA_DRIVERS_DIR=$t/absent ${exe} --dry-run'' + ) + # what a person has set is kept, and the directory of Windows' libraries comes first + (scenario "dry-run-wslg-gpu-own-env" 0 + ''machine ${wslg} RUN_KDE_DXG_DEVICE=$t/dxg "RUN_KDE_HOST_RENDERER=D3D12 (AMD Radeon RX 7800 XT)" GALLIUM_DRIVER=zink LD_LIBRARY_PATH=/opt/lib LIBVA_DRIVER_NAME=radeonsi ${exe} --dry-run'' + ) (scenario "dry-run-wslg-no-gpu" 0 "machine ${wslg} RUN_KDE_DXG_DEVICE=$t/absent RUN_KDE_HOST_RENDERER= ${exe} --dry-run" ) @@ -86,6 +95,9 @@ in (scenario "doctor-wslg-gpu" 0 ''machine ${wslg} RUN_KDE_DXG_DEVICE=$t/dxg "RUN_KDE_HOST_RENDERER=D3D12 (AMD Radeon RX 7800 XT)" RUN_KDE_WSLCONFIG=$t/none ${exe} --doctor'' ) + (scenario "doctor-wslg-gpu-no-driver" 0 + ''machine ${wslg} RUN_KDE_DXG_DEVICE=$t/dxg "RUN_KDE_HOST_RENDERER=D3D12 (AMD Radeon RX 7800 XT)" RUN_KDE_WSL_LIB_DIR=$t/absent RUN_KDE_VA_DRIVERS_DIR=$t/absent RUN_KDE_WSLCONFIG=$t/none ${exe} --doctor'' + ) (scenario "doctor-wsl-no-kvm" 1 "machine WAYLAND_DISPLAY=w RUN_KDE_OSRELEASE=$t/osrelease-wsl RUN_KDE_KVM_DEVICE=$t/absent RUN_KDE_CPUINFO=$t/cpuinfo-none RUN_KDE_MEMINFO=$t/meminfo-small RUN_KDE_DXG_DEVICE=$t/absent RUN_KDE_HOST_RENDERER= RUN_KDE_WSLCONFIG=$t/none ${exe} --doctor" ) diff --git a/nix/tests/run-kde/doctor-wslg-gpu-no-driver.expected b/nix/tests/run-kde/doctor-wslg-gpu-no-driver.expected new file mode 100644 index 00000000..fb3197b6 --- /dev/null +++ b/nix/tests/run-kde/doctor-wslg-gpu-no-driver.expected @@ -0,0 +1,20 @@ +run-kde: what the guest needs, and what this machine has + ✓ this is WSL 2 (5.15.167.4-microsoft-standard-WSL2) + ✓ nix is installed + ✓ KVM: @T@/kvm can be used + ✓ FUSE: @T@/fuse can be opened + ✓ virtiofsd: starts as this user and serves a directory + ✓ memory: this machine has 16384 MB, and the guest gets 8192 MB + ✓ cores: this machine has 12, and the guest gets 6 + ✓ disk: 100 GB are free for the Nix store (@T@) + ✓ a window can be shown (WAYLAND_DISPLAY=wayland-0 DISPLAY=:0) + ! Windows interop: mstsc.exe or cmdkey.exe cannot be run, which --rdp needs + on NixOS-WSL, wsl.interop.includePath = true; (and wsl.interop.register = true;) + ✓ GPU: @T@/dxg is there + ✓ GPU: the host's OpenGL renderer is D3D12 (AMD Radeon RX 7800 XT) + ! GPU: there is no @T@/absent, so Mesa cannot take the Windows driver (d3d12) + run `wsl --update` in PowerShell, and install the Windows GPU driver with WSL support + ! GPU: there is no d3d12 video driver in @T@/absent, so KRDP encodes H.264 on the guest's CPUs + on NixOS-WSL, wsl.useWindowsDriver = true; with a Mesa that is built with video codecs + ✓ the guest will use the GPU: @T@/dxg is there and the OpenGL of the host is D3D12 (AMD Radeon RX 7800 XT) +run-kde: 0 error(s), 3 warning(s) diff --git a/nix/tests/run-kde/doctor-wslg-gpu.expected b/nix/tests/run-kde/doctor-wslg-gpu.expected index 9828f4e7..c108754d 100644 --- a/nix/tests/run-kde/doctor-wslg-gpu.expected +++ b/nix/tests/run-kde/doctor-wslg-gpu.expected @@ -12,5 +12,7 @@ run-kde: what the guest needs, and what this machine has on NixOS-WSL, wsl.interop.includePath = true; (and wsl.interop.register = true;) ✓ GPU: @T@/dxg is there ✓ GPU: the host's OpenGL renderer is D3D12 (AMD Radeon RX 7800 XT) + ✓ GPU: the Windows driver is in @T@/wsl-lib + ✓ GPU: the video driver of Mesa for it (d3d12) is in @T@/va ✓ the guest will use the GPU: @T@/dxg is there and the OpenGL of the host is D3D12 (AMD Radeon RX 7800 XT) run-kde: 0 error(s), 1 warning(s) diff --git a/nix/tests/run-kde/dry-run-wslg-gpu-no-video-driver.expected b/nix/tests/run-kde/dry-run-wslg-gpu-no-video-driver.expected new file mode 100644 index 00000000..ff545a24 --- /dev/null +++ b/nix/tests/run-kde/dry-run-wslg-gpu-no-video-driver.expected @@ -0,0 +1,14 @@ +run-kde would start run-kde-x86_64: + host: WSL 2, KVM yes + display: gtk + gpu: on, because @T@/dxg is there and the OpenGL of the host is D3D12 (AMD Radeon RX 7800 XT) + environment: + NIX_DISK_IMAGE=/state/komorebi-run-kde/disk.qcow2 + KOMOREBI_REPO=@T@/repo + RUN_KDE_STATE=/state/komorebi-run-kde + RUN_KDE_MEMORY=8192 + QEMU_NET_OPTS=hostfwd=tcp:127.0.0.1:2223-:22,hostfwd=tcp:127.0.0.1:3389-:3389 + QEMU_OPTS=-cpu host -display gtk,gl=on -vga none -device virtio-vga-gl,xres=1280,yres=800 + GALLIUM_DRIVER=d3d12 + LD_LIBRARY_PATH=@T@/wsl-lib + command: /nix/store/fake-nixos-vm/bin/run-komorebi-demo-vm -m 8192 -smp 6 diff --git a/nix/tests/run-kde/dry-run-wslg-gpu-own-env.expected b/nix/tests/run-kde/dry-run-wslg-gpu-own-env.expected new file mode 100644 index 00000000..948baf1e --- /dev/null +++ b/nix/tests/run-kde/dry-run-wslg-gpu-own-env.expected @@ -0,0 +1,16 @@ +run-kde would start run-kde-x86_64: + host: WSL 2, KVM yes + display: gtk + gpu: on, because @T@/dxg is there and the OpenGL of the host is D3D12 (AMD Radeon RX 7800 XT) + environment: + NIX_DISK_IMAGE=/state/komorebi-run-kde/disk.qcow2 + KOMOREBI_REPO=@T@/repo + RUN_KDE_STATE=/state/komorebi-run-kde + RUN_KDE_MEMORY=8192 + QEMU_NET_OPTS=hostfwd=tcp:127.0.0.1:2223-:22,hostfwd=tcp:127.0.0.1:3389-:3389 + QEMU_OPTS=-cpu host -display gtk,gl=on -vga none -device virtio-vga-gl,xres=1280,yres=800 + GALLIUM_DRIVER=zink + LD_LIBRARY_PATH=@T@/wsl-lib:/opt/lib + LIBVA_DRIVER_NAME=radeonsi + LIBVA_DRIVERS_PATH=@T@/va + command: /nix/store/fake-nixos-vm/bin/run-komorebi-demo-vm -m 8192 -smp 6 diff --git a/nix/tests/run-kde/dry-run-wslg-gpu.expected b/nix/tests/run-kde/dry-run-wslg-gpu.expected index 0c92fbf9..549c2238 100644 --- a/nix/tests/run-kde/dry-run-wslg-gpu.expected +++ b/nix/tests/run-kde/dry-run-wslg-gpu.expected @@ -9,4 +9,8 @@ run-kde would start run-kde-x86_64: RUN_KDE_MEMORY=8192 QEMU_NET_OPTS=hostfwd=tcp:127.0.0.1:2223-:22,hostfwd=tcp:127.0.0.1:3389-:3389 QEMU_OPTS=-cpu host -display gtk,gl=on -vga none -device virtio-vga-gl,xres=1280,yres=800 + GALLIUM_DRIVER=d3d12 + LD_LIBRARY_PATH=@T@/wsl-lib + LIBVA_DRIVER_NAME=d3d12 + LIBVA_DRIVERS_PATH=@T@/va command: /nix/store/fake-nixos-vm/bin/run-komorebi-demo-vm -m 8192 -smp 6 diff --git a/scripts/run-kde b/scripts/run-kde old mode 100755 new mode 100644 index fc4c2fd3..123cffe6 --- a/scripts/run-kde +++ b/scripts/run-kde @@ -59,6 +59,8 @@ osrelease=${RUN_KDE_OSRELEASE:-/proc/sys/kernel/osrelease} sys_module_dir=${RUN_KDE_SYS_MODULE_DIR:-/sys/module} store_dir=${RUN_KDE_STORE_DIR:-/nix} wslconfig=${RUN_KDE_WSLCONFIG:-} +wsl_lib_dir=${RUN_KDE_WSL_LIB_DIR:-/usr/lib/wsl/lib} +va_drivers_dir=${RUN_KDE_VA_DRIVERS_DIR:-/run/opengl-driver/lib/dri} mode=run display=auto @@ -276,6 +278,26 @@ find_virtiofsd() { fi } +# What the host's GPU needs of the process that uses it, on WSL 2. Mesa has to be told to take the +# d3d12 driver of Windows, whose libraries are in a directory of their own, and the libva that +# virglrenderer takes the host's video encoder from has to be told which driver that is. Without +# the first OpenGL is llvmpipe, and without the second the guest's VA-API lists no encoder, so KRDP +# encodes H.264 on the guest's CPUs. What a person has set is left as it is. +host_env=() +if [ "$wsl" = 1 ] && [ "$gpu_on" = 1 ]; then + host_env+=("GALLIUM_DRIVER=${GALLIUM_DRIVER:-d3d12}") + if [ -d "$wsl_lib_dir" ]; then + case ":${LD_LIBRARY_PATH:-}:" in + *":$wsl_lib_dir:"*) ;; + *) host_env+=("LD_LIBRARY_PATH=$wsl_lib_dir${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}") ;; + esac + fi + if [ -e "$va_drivers_dir/d3d12_drv_video.so" ]; then + host_env+=("LIBVA_DRIVER_NAME=${LIBVA_DRIVER_NAME:-d3d12}") + host_env+=("LIBVA_DRIVERS_PATH=${LIBVA_DRIVERS_PATH:-$va_drivers_dir}") + fi +fi + # --------------------------------------------------------------------------------------------- # the checks @@ -474,6 +496,20 @@ check_everything() { *) ok "GPU: the host's OpenGL renderer is $renderer" ;; esac fi + if [ "$wsl" = 1 ] && [ -n "$gpu_device" ]; then + if [ -d "$wsl_lib_dir" ]; then + ok "GPU: the Windows driver is in $wsl_lib_dir" + else + warn "GPU: there is no $wsl_lib_dir, so Mesa cannot take the Windows driver (d3d12)" \ + "run \`wsl --update\` in PowerShell, and install the Windows GPU driver with WSL support" + fi + if [ -e "$va_drivers_dir/d3d12_drv_video.so" ]; then + ok "GPU: the video driver of Mesa for it (d3d12) is in $va_drivers_dir" + else + warn "GPU: there is no d3d12 video driver in $va_drivers_dir, so KRDP encodes H.264 on the guest's CPUs" \ + "on NixOS-WSL, wsl.useWindowsDriver = true; with a Mesa that is built with video codecs" + fi + fi if [ "$gpu_on" = 1 ]; then ok "the guest will use the GPU: $gpu_why" else @@ -564,6 +600,7 @@ if [ "$mode" = dry-run ]; then echo " RUN_KDE_MEMORY=$guest_memory" echo " QEMU_NET_OPTS=$net_opts" echo " QEMU_OPTS=${qemu_opts[*]}" + for var in "${host_env[@]}"; do echo " $var"; done echo " command: ${runner_bin:-} ${runner_args[*]}" exit 0 fi @@ -633,6 +670,10 @@ export RUN_KDE_STATE=$state_dir export RUN_KDE_MEMORY=$guest_memory export QEMU_NET_OPTS=$net_opts export QEMU_OPTS="${qemu_opts[*]}${RUN_KDE_QEMU_OPTS:+ $RUN_KDE_QEMU_OPTS}" +for var in "${host_env[@]}"; do + # shellcheck disable=SC2163 # the variable named by what `var` holds, as NAME=value + export "$var" +done # virtiofsd says a great deal about being run by a user and not by root, none of it a problem export RUST_LOG=${RUST_LOG:-error}