mirror of
https://github.com/LGUG2Z/komorebi.git
synced 2026-11-20 15:41:38 +01:00
fix(capture): scale a display H.264 cannot take
An H.264 encoder refuses frames bigger than 4096 x 2304 (either way round). On a 5120 x 2160 display the sink writer failed at SetInputMediaType with 0xC00D36B4 (MF_E_INVALIDMEDIATYPE), reported as "the encoder cannot take frames of this format", and no display could be recorded. segment::h264_size gives the size to encode at: the display's own, made even, when H.264 can take it, and otherwise the largest even size of the same shape that it can (5120 x 2160 is recorded at 4096 x 1728). When that is not the display's size, a Direct3D 11 video processor scales each frame on the GPU before it is handed to the encoder. Displays that fit are untouched. Recorded and decoded on a 5120 x 2160 display: all three screens, undistorted.
This commit is contained in:
1 parent
8a203f0da7
commit
b38a6f4dc6
2 files changed
+214
-28
No files matched your search
@@ -13,7 +13,7 @@
|
||||
//! | [`FrameLog`] | the `frames-<display>[-<n>].csv` file: when each frame was shown and received, and a row for each repeat |
|
||||
//! | [`keepalive_due`], [`KEEPALIVE_AFTER`], [`KEEPALIVE_SOURCE_TIMEOUT_NS`] | when a display that has shown nothing new needs its last frame repeated |
|
||||
//! | [`Latency`] | the clock-unification check, as [`LatencyStats`] |
|
||||
//! | [`even`], [`h264_bitrate`], [`rebased_100ns`] | the encoding's size and bitrate, and the file's unit of time |
|
||||
//! | [`even`], [`h264_size`], [`h264_bitrate`], [`rebased_100ns`] | the encoding's size and bitrate, and the file's unit of time |
|
||||
//!
|
||||
//! # Who uses it
|
||||
//!
|
||||
@@ -82,6 +82,45 @@ pub fn even(value: f64) -> usize {
|
||||
value & !1
|
||||
}
|
||||
|
||||
/// The longest side H.264 can have: level 5.2, which is as far as hardware encoders go, allows
|
||||
/// 4096 x 2304.
|
||||
const H264_LONG_SIDE: f64 = 4096.0;
|
||||
/// The longest the shorter side of a frame can be: see [`H264_LONG_SIDE`].
|
||||
const H264_SHORT_SIDE: f64 = 2304.0;
|
||||
|
||||
/// Returns the size to encode a screen of `width` x `height` pixels at: the screen's own size
|
||||
/// (made even) when H.264 can take it, and otherwise the largest even size with the same shape
|
||||
/// that it can.
|
||||
///
|
||||
/// An H.264 encoder refuses a frame bigger than 4096 x 2304 (either way round), which a 5K
|
||||
/// ultrawide or an 8K display is. Only the Windows backend calls it: the macOS encoder scales for
|
||||
/// itself.
|
||||
///
|
||||
/// # Example
|
||||
///
|
||||
/// ```rust
|
||||
/// use komorebi_recorder::capture::segment::h264_size;
|
||||
///
|
||||
/// assert_eq!(h264_size(1920, 1080), (1920, 1080));
|
||||
/// assert_eq!(h264_size(3840, 2160), (3840, 2160));
|
||||
/// // Odd sizes are made even, as ever.
|
||||
/// assert_eq!(h264_size(1919, 1081), (1918, 1080));
|
||||
/// // A 5K ultrawide is brought down to the width H.264 allows.
|
||||
/// assert_eq!(h264_size(5120, 2160), (4096, 1728));
|
||||
/// // And so is a tall one.
|
||||
/// assert_eq!(h264_size(2160, 5120), (1728, 4096));
|
||||
/// assert_eq!(h264_size(7680, 4320), (4096, 2304));
|
||||
/// ```
|
||||
#[must_use]
|
||||
pub fn h264_size(width: usize, height: usize) -> (usize, usize) {
|
||||
let (width, height) = (width as f64, height as f64);
|
||||
let (long, short) = (width.max(height), width.min(height));
|
||||
let scale = (H264_LONG_SIDE / long)
|
||||
.min(H264_SHORT_SIDE / short)
|
||||
.min(1.0);
|
||||
(even(width * scale), even(height * scale))
|
||||
}
|
||||
|
||||
/// Returns the bitrate to encode a screen of this size at, in bits per second.
|
||||
///
|
||||
/// Screen content is mostly flat and changes in bursts, so this is about a tenth of a bit per pixel
|
||||
|
||||
@@ -69,6 +69,7 @@ use super::segment::Latency;
|
||||
use super::segment::Placement;
|
||||
use super::segment::even;
|
||||
use super::segment::h264_bitrate;
|
||||
use super::segment::h264_size;
|
||||
use super::segment::keepalive_due;
|
||||
use super::video_file_name;
|
||||
use crate::clock;
|
||||
@@ -95,13 +96,29 @@ use windows::Win32::Graphics::Direct3D11::D3D11_BOX;
|
||||
use windows::Win32::Graphics::Direct3D11::D3D11_CREATE_DEVICE_BGRA_SUPPORT;
|
||||
use windows::Win32::Graphics::Direct3D11::D3D11_CREATE_DEVICE_VIDEO_SUPPORT;
|
||||
use windows::Win32::Graphics::Direct3D11::D3D11_SDK_VERSION;
|
||||
use windows::Win32::Graphics::Direct3D11::D3D11_TEX2D_VPIV;
|
||||
use windows::Win32::Graphics::Direct3D11::D3D11_TEX2D_VPOV;
|
||||
use windows::Win32::Graphics::Direct3D11::D3D11_TEXTURE2D_DESC;
|
||||
use windows::Win32::Graphics::Direct3D11::D3D11_USAGE_DEFAULT;
|
||||
use windows::Win32::Graphics::Direct3D11::D3D11_VIDEO_FRAME_FORMAT_PROGRESSIVE;
|
||||
use windows::Win32::Graphics::Direct3D11::D3D11_VIDEO_PROCESSOR_CONTENT_DESC;
|
||||
use windows::Win32::Graphics::Direct3D11::D3D11_VIDEO_PROCESSOR_INPUT_VIEW_DESC;
|
||||
use windows::Win32::Graphics::Direct3D11::D3D11_VIDEO_PROCESSOR_INPUT_VIEW_DESC_0;
|
||||
use windows::Win32::Graphics::Direct3D11::D3D11_VIDEO_PROCESSOR_OUTPUT_VIEW_DESC;
|
||||
use windows::Win32::Graphics::Direct3D11::D3D11_VIDEO_PROCESSOR_OUTPUT_VIEW_DESC_0;
|
||||
use windows::Win32::Graphics::Direct3D11::D3D11_VIDEO_PROCESSOR_STREAM;
|
||||
use windows::Win32::Graphics::Direct3D11::D3D11_VIDEO_USAGE_PLAYBACK_NORMAL;
|
||||
use windows::Win32::Graphics::Direct3D11::D3D11_VPIV_DIMENSION_TEXTURE2D;
|
||||
use windows::Win32::Graphics::Direct3D11::D3D11_VPOV_DIMENSION_TEXTURE2D;
|
||||
use windows::Win32::Graphics::Direct3D11::D3D11CreateDevice;
|
||||
use windows::Win32::Graphics::Direct3D11::ID3D11Device;
|
||||
use windows::Win32::Graphics::Direct3D11::ID3D11DeviceContext;
|
||||
use windows::Win32::Graphics::Direct3D11::ID3D11Multithread;
|
||||
use windows::Win32::Graphics::Direct3D11::ID3D11Texture2D;
|
||||
use windows::Win32::Graphics::Direct3D11::ID3D11VideoContext;
|
||||
use windows::Win32::Graphics::Direct3D11::ID3D11VideoDevice;
|
||||
use windows::Win32::Graphics::Direct3D11::ID3D11VideoProcessor;
|
||||
use windows::Win32::Graphics::Direct3D11::ID3D11VideoProcessorEnumerator;
|
||||
use windows::Win32::Graphics::Dxgi::Common::DXGI_FORMAT_B8G8R8A8_UNORM;
|
||||
use windows::Win32::Graphics::Dxgi::Common::DXGI_SAMPLE_DESC;
|
||||
use windows::Win32::Graphics::Dxgi::IDXGIDevice;
|
||||
@@ -226,6 +243,34 @@ impl Gpu {
|
||||
width: u32,
|
||||
height: u32,
|
||||
) -> eyre::Result<ID3D11Texture2D> {
|
||||
let texture = self.texture(width, height)?;
|
||||
let region = D3D11_BOX {
|
||||
left: 0,
|
||||
top: 0,
|
||||
front: 0,
|
||||
right: width,
|
||||
bottom: height,
|
||||
back: 1,
|
||||
};
|
||||
// SAFETY: a copy between textures of the same device, and the immediate context is
|
||||
// protected for use from many threads
|
||||
unsafe {
|
||||
self.context.CopySubresourceRegion(
|
||||
&texture,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
source,
|
||||
0,
|
||||
Some(&raw const region),
|
||||
);
|
||||
}
|
||||
Ok(texture)
|
||||
}
|
||||
|
||||
/// An empty texture of our own, a frame's size.
|
||||
fn texture(&self, width: u32, height: u32) -> eyre::Result<ID3D11Texture2D> {
|
||||
let description = D3D11_TEXTURE2D_DESC {
|
||||
Width: width,
|
||||
Height: height,
|
||||
@@ -241,34 +286,110 @@ impl Gpu {
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
// SAFETY: the texture is created from a valid description; the copy is between textures
|
||||
// of the same device, and the immediate context is protected for use from many threads
|
||||
// SAFETY: the texture is created from a valid description
|
||||
unsafe {
|
||||
let mut texture = None;
|
||||
self.device
|
||||
.CreateTexture2D(&raw const description, None, Some(&raw mut texture))
|
||||
.wrap_err("could not allocate a frame")?;
|
||||
let texture = texture.ok_or_else(|| eyre::Report::msg("no frame texture"))?;
|
||||
texture.ok_or_else(|| eyre::Report::msg("no frame texture"))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let region = D3D11_BOX {
|
||||
left: 0,
|
||||
top: 0,
|
||||
front: 0,
|
||||
right: width,
|
||||
bottom: height,
|
||||
back: 1,
|
||||
/// Scales frames down on the GPU, for a display too big for H.264 (see [`h264_size`]).
|
||||
struct Scaler {
|
||||
device: ID3D11VideoDevice,
|
||||
context: ID3D11VideoContext,
|
||||
enumerator: ID3D11VideoProcessorEnumerator,
|
||||
processor: ID3D11VideoProcessor,
|
||||
}
|
||||
|
||||
impl Scaler {
|
||||
fn new(gpu: &Gpu, from: (u32, u32), to: (u32, u32)) -> eyre::Result<Self> {
|
||||
let description = D3D11_VIDEO_PROCESSOR_CONTENT_DESC {
|
||||
InputFrameFormat: D3D11_VIDEO_FRAME_FORMAT_PROGRESSIVE,
|
||||
InputWidth: from.0,
|
||||
InputHeight: from.1,
|
||||
OutputWidth: to.0,
|
||||
OutputHeight: to.1,
|
||||
Usage: D3D11_VIDEO_USAGE_PLAYBACK_NORMAL,
|
||||
..Default::default()
|
||||
};
|
||||
// SAFETY: creation calls on the capture's device with a valid description
|
||||
unsafe {
|
||||
let device: ID3D11VideoDevice = gpu.device.cast()?;
|
||||
let context: ID3D11VideoContext = gpu.context.cast()?;
|
||||
let enumerator = device
|
||||
.CreateVideoProcessorEnumerator(&raw const description)
|
||||
.wrap_err("could not create a video processor to scale frames")?;
|
||||
let processor = device
|
||||
.CreateVideoProcessor(&enumerator, 0)
|
||||
.wrap_err("could not create a video processor to scale frames")?;
|
||||
Ok(Self {
|
||||
device,
|
||||
context,
|
||||
enumerator,
|
||||
processor,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// Scales all of `source` to fill `target`.
|
||||
fn scale(&self, source: &ID3D11Texture2D, target: &ID3D11Texture2D) -> eyre::Result<()> {
|
||||
let input_description = D3D11_VIDEO_PROCESSOR_INPUT_VIEW_DESC {
|
||||
FourCC: 0,
|
||||
ViewDimension: D3D11_VPIV_DIMENSION_TEXTURE2D,
|
||||
Anonymous: D3D11_VIDEO_PROCESSOR_INPUT_VIEW_DESC_0 {
|
||||
Texture2D: D3D11_TEX2D_VPIV {
|
||||
MipSlice: 0,
|
||||
ArraySlice: 0,
|
||||
},
|
||||
},
|
||||
};
|
||||
let output_description = D3D11_VIDEO_PROCESSOR_OUTPUT_VIEW_DESC {
|
||||
ViewDimension: D3D11_VPOV_DIMENSION_TEXTURE2D,
|
||||
Anonymous: D3D11_VIDEO_PROCESSOR_OUTPUT_VIEW_DESC_0 {
|
||||
Texture2D: D3D11_TEX2D_VPOV { MipSlice: 0 },
|
||||
},
|
||||
};
|
||||
|
||||
// SAFETY: views of textures of the device the processor was made on; the stream holds
|
||||
// its own reference to the input view for the call
|
||||
unsafe {
|
||||
let mut input = None;
|
||||
self.device
|
||||
.CreateVideoProcessorInputView(
|
||||
source,
|
||||
&self.enumerator,
|
||||
&raw const input_description,
|
||||
Some(&raw mut input),
|
||||
)
|
||||
.wrap_err("could not read a frame to scale it")?;
|
||||
let mut output = None;
|
||||
self.device
|
||||
.CreateVideoProcessorOutputView(
|
||||
target,
|
||||
&self.enumerator,
|
||||
&raw const output_description,
|
||||
Some(&raw mut output),
|
||||
)
|
||||
.wrap_err("could not write a scaled frame")?;
|
||||
let output = output.ok_or_else(|| eyre::Report::msg("no scaled frame view"))?;
|
||||
|
||||
let stream = D3D11_VIDEO_PROCESSOR_STREAM {
|
||||
Enable: true.into(),
|
||||
pInputSurface: std::mem::ManuallyDrop::new(input),
|
||||
..Default::default()
|
||||
};
|
||||
self.context.CopySubresourceRegion(
|
||||
&texture,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
source,
|
||||
0,
|
||||
Some(&raw const region),
|
||||
);
|
||||
Ok(texture)
|
||||
let streams = [stream];
|
||||
let blt = self
|
||||
.context
|
||||
.VideoProcessorBlt(&self.processor, &output, 0, &streams);
|
||||
// takes back the reference the stream held, so that the view is released
|
||||
let [stream] = streams;
|
||||
drop(std::mem::ManuallyDrop::into_inner(stream.pInputSurface));
|
||||
blt.wrap_err("could not scale a frame")
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -419,7 +540,10 @@ struct Shared {
|
||||
stats: Arc<CaptureStats>,
|
||||
state: Mutex<State>,
|
||||
gpu: Gpu,
|
||||
/// The size the segment is recorded at, in pixels (the source's, made even).
|
||||
/// Brings frames down to the size they are recorded at, when the source is too big for H.264.
|
||||
scaler: Option<Scaler>,
|
||||
/// The size the segment is recorded at, in pixels (the source's, made even, and scaled down
|
||||
/// if H.264 cannot take it).
|
||||
width: u32,
|
||||
height: u32,
|
||||
/// The size of the display when the capture began: a frame of any other size means it changed.
|
||||
@@ -483,7 +607,19 @@ fn frame_into_file(
|
||||
let access: IDirect3DDxgiInterfaceAccess = surface.cast()?;
|
||||
// SAFETY: a capture surface is a texture of the capture's device
|
||||
let source: ID3D11Texture2D = unsafe { access.GetInterface()? };
|
||||
let texture = shared.gpu.copy(&source, shared.width, shared.height)?;
|
||||
let texture = match &shared.scaler {
|
||||
Some(scaler) => {
|
||||
let whole = shared.gpu.copy(
|
||||
&source,
|
||||
shared.source.Width as u32,
|
||||
shared.source.Height as u32,
|
||||
)?;
|
||||
let scaled = shared.gpu.texture(shared.width, shared.height)?;
|
||||
scaler.scale(&whole, &scaled)?;
|
||||
scaled
|
||||
}
|
||||
None => shared.gpu.copy(&source, shared.width, shared.height)?,
|
||||
};
|
||||
let buffer = buffer_of(&texture)?;
|
||||
|
||||
let mut state = shared.state.lock();
|
||||
@@ -602,12 +738,22 @@ impl DisplayCapture {
|
||||
crate::platform::media::start()?;
|
||||
let item = item_for(display)?;
|
||||
let size = item.Size()?;
|
||||
let (width, height) = (
|
||||
u32::try_from(even(f64::from(size.Width)))?,
|
||||
u32::try_from(even(f64::from(size.Height)))?,
|
||||
);
|
||||
let evened = (even(f64::from(size.Width)), even(f64::from(size.Height)));
|
||||
let (width, height) = h264_size(evened.0, evened.1);
|
||||
let (width, height) = (u32::try_from(width)?, u32::try_from(height)?);
|
||||
|
||||
let gpu = Gpu::new()?;
|
||||
// a display H.264 cannot take is scaled down; any other has the top left of it recorded
|
||||
// (which is all of it, unless its size is odd)
|
||||
let scaler = if (width as usize, height as usize) == evened {
|
||||
None
|
||||
} else {
|
||||
Some(Scaler::new(
|
||||
&gpu,
|
||||
(size.Width as u32, size.Height as u32),
|
||||
(width, height),
|
||||
)?)
|
||||
};
|
||||
let file = video_file_name(display.id, segment);
|
||||
let frames_file = frames_file_name(display.id, segment);
|
||||
let path = dir.join(&file);
|
||||
@@ -627,6 +773,7 @@ impl DisplayCapture {
|
||||
finished: false,
|
||||
}),
|
||||
gpu,
|
||||
scaler,
|
||||
width,
|
||||
height,
|
||||
source: size,
|
||||
|
||||
Reference in new issue
Block a user