Files
komorebi/nix/linux/dev/virglrenderer-chroma-format.patch
T
LGUG2Z f01853ceca refactor(repo): move the KDE and macOS nix files under nix/ (pure move)
komorebi-for-kde/nix/ -> nix/linux/
komorebi-for-mac/nix/ -> nix/macos/

The nix expressions of the platforms (modules, option files, VM test suites, kwin effects,
the darwin module) sit next to the existing nix/workspace-src.nix. Renames only (252
files); the flakes that import them are replaced by one root flake later in this stack, so
nix does not evaluate at this revision.
2026-11-05 15:03:27 -08:00

55 lines
2.9 KiB
Diff

virglrenderer: read the chroma format of a video codec in the guest's numbering
The guest's Mesa (26.2.5) sends a video codec's chroma format as the number of its own
pipe_video_chroma_format, which since the enum moved to u_format.h is NONE, 400,
420, 422, 440, 444 (0 to 5). The enum of virglrenderer, generated by
u_format_table.py, is 400, 420, 422, 444, 440, NONE, so 4:2:0, which is 2, is read as
4:2:2 and VA-API's encoder, which has no 4:2:2, refuses the codec: "format 0x2 not
supported, supported formats: 0x1". The guest's encode then returns no data at all.
This reads the number as Mesa's. It is for a guest whose Mesa numbers it that way: a
guest whose Mesa still numbers it as virglrenderer does would be read wrongly.
diff --git a/src/vrend/vrend_decode.c b/src/vrend/vrend_decode.c
index 7774a25..c6eb8f2 100644
--- a/src/vrend/vrend_decode.c
+++ b/src/vrend/vrend_decode.c
@@ -1766,6 +1766,26 @@ static int vrend_decode_get_pipe_resource_layout(struct vrend_context *ctx, cons
#ifdef ENABLE_VIDEO
/* video codec related functions */
+/*
+ * The chroma format of a codec comes from the guest as the number of its own enum, and the enum
+ * of Mesa since its pipe_video_chroma_format moved to u_format.h is numbered NONE, 400, 420, 422,
+ * 440, 444 (0 to 5), where this one, which u_format_table.py generates, is 400, 420, 422, 444, 440,
+ * NONE. Taken as is, 4:2:0, which is 2, is 4:2:2 here, and the host's encoder, which has no 4:2:2,
+ * refuses the codec ("format 0x2 not supported, supported formats: 0x1"), after which the guest
+ * encodes nothing.
+ */
+static uint32_t vrend_decode_chroma_format(uint32_t guest_format)
+{
+ switch (guest_format) {
+ case 1: return PIPE_VIDEO_CHROMA_FORMAT_400;
+ case 2: return PIPE_VIDEO_CHROMA_FORMAT_420;
+ case 3: return PIPE_VIDEO_CHROMA_FORMAT_422;
+ case 4: return PIPE_VIDEO_CHROMA_FORMAT_440;
+ case 5: return PIPE_VIDEO_CHROMA_FORMAT_444;
+ default: return PIPE_VIDEO_CHROMA_FORMAT_NONE;
+ }
+}
+
static int vrend_decode_create_video_codec(struct vrend_context *ctx,
const uint32_t *buf,
uint32_t length)
@@ -1778,7 +1798,8 @@ static int vrend_decode_create_video_codec(struct vrend_context *ctx,
uint32_t handle = get_buf_entry(buf, VIRGL_CREATE_VIDEO_CODEC_HANDLE);
uint32_t profile = get_buf_entry(buf, VIRGL_CREATE_VIDEO_CODEC_PROFILE);
uint32_t entrypoint = get_buf_entry(buf, VIRGL_CREATE_VIDEO_CODEC_ENTRYPOINT);
- uint32_t chroma_fmt = get_buf_entry(buf, VIRGL_CREATE_VIDEO_CODEC_CHROMA_FMT);
+ uint32_t chroma_fmt = vrend_decode_chroma_format(
+ get_buf_entry(buf, VIRGL_CREATE_VIDEO_CODEC_CHROMA_FMT));
uint32_t level = get_buf_entry(buf, VIRGL_CREATE_VIDEO_CODEC_LEVEL);
uint32_t width = get_buf_entry(buf, VIRGL_CREATE_VIDEO_CODEC_WIDTH);
uint32_t height = get_buf_entry(buf, VIRGL_CREATE_VIDEO_CODEC_HEIGHT);