From 11f028b8c98a4d1aedf9fd3e4c6b5e2542d785e4 Mon Sep 17 00:00:00 2001 From: LGUG2Z Date: Wed, 28 Oct 2026 08:55:58 -0700 Subject: [PATCH] feat(capture): record at the display's refresh rate, and give each frame its real duration Windows Graphics Capture holds a capture to 60 frames a second unless it is given a minimum update interval, so a 120 Hz display was recorded at half its rate and a one-refresh flash could fall between two frames. ScreenCaptureKit was asked for at most 30 (the default --fps), so macOS lost frames the same way. --fps now means the most frames a second, and 0, the new default, means no cap: Windows sets the session's minimum update interval to a millisecond (or a frame at --fps), and macOS sets a zero minimum frame interval. The encoder is told 60 fps when there is no cap. On Windows each sample was also stamped to last 1/fps whatever the gap to the next frame. A frame is now held until the next arrives and given the real time between them; the last one still gets a frame at the nominal rate. macOS passes ScreenCaptureKit's own sample timing through, so needs no change. Checked on a Mac with a 120 Hz display and komorebi for Mac running: with no cap a display is recorded a frame every 8.3 ms, and --fps 30 holds it to about 30 a second. The video holds as many frames as the frame log has rows, and the state komorebi was recorded in matches what the frames show. The macOS encoder never reads fps, so CaptureConfig::fps now says it reaches only the Windows one. --- docs/usage/recording-a-bug-repro.md | 2 +- komorebi-recorder/src/capture/mod.rs | 31 +++++++---------- komorebi-recorder/src/capture/sck.rs | 16 ++++++--- komorebi-recorder/src/capture/segment.rs | 43 ++++++++++++++++++++++++ komorebi-recorder/src/capture/wgc.rs | 36 +++++++++++++++++--- komorebi-recorder/src/cli/record.rs | 13 +++---- komorebi-recorder/src/testkit.rs | 5 +-- komorebi-recorder/src/viewer/rules.rs | 2 +- 8 files changed, 110 insertions(+), 38 deletions(-) diff --git a/docs/usage/recording-a-bug-repro.md b/docs/usage/recording-a-bug-repro.md index 59dd454a..d3f816d7 100644 --- a/docs/usage/recording-a-bug-repro.md +++ b/docs/usage/recording-a-bug-repro.md @@ -93,7 +93,7 @@ Useful options: | `--no-oslog`, `--no-komorebi` | Skip the OS trace, or skip touching komorebi entirely | | `--no-video` | Do not capture the screen; record everything else on the same clock (see below) | | `--no-probe-windows` | Windows: do not record what Windows says about each window event, which is on by default (see below) | -| `--fps N`, `--retina`, `--codec hevc\|h264` | Video settings (30 fps cap, logical points, HEVC by default) | +| `--fps N`, `--retina`, `--codec hevc\|h264` | Video settings (display-rate capture (`--fps 0`, the default; N caps it), logical points, HEVC by default) | ### Marking where the bug is diff --git a/komorebi-recorder/src/capture/mod.rs b/komorebi-recorder/src/capture/mod.rs index acc6c173..292febe0 100644 --- a/komorebi-recorder/src/capture/mod.rs +++ b/komorebi-recorder/src/capture/mod.rs @@ -139,7 +139,8 @@ impl std::fmt::Display for Codec { /// [`DisplayManager::new`](manager::DisplayManager::new) takes it once and hands it to /// [`CaptureBackend::start`] for each segment. /// -/// The default is 30 frames a second, not retina, and [`Codec::default`]. +/// The default is no frame rate cap (frames as fast as the display refreshes), not retina, and +/// [`Codec::default`]. /// /// # Example /// @@ -147,19 +148,21 @@ impl std::fmt::Display for Codec { /// use komorebi_recorder::capture::{CaptureConfig, Codec}; /// /// let config = CaptureConfig::default(); -/// assert_eq!(config.fps, 30); +/// assert_eq!(config.fps, 0); /// assert!(!config.retina); /// assert_eq!(config.codec, Codec::default()); /// ``` -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Default)] pub struct CaptureConfig { - /// The frame rate the capture is configured for. `0` is treated as `1`. + /// The most frames a second to capture. `0` means no cap: as many as the display refreshes. /// - /// On a Mac this is an upper bound: `ScreenCaptureKit` is asked for no more than `fps` frames a - /// second, and emits frames only when the content changes. On Windows it is the nominal rate - /// declared to the encoder. It sets the bitrate (see [`segment::h264_bitrate`]), the key frame - /// spacing and the duration given to the last frame, and does not limit how often frames - /// arrive. + /// Both backends treat this as an upper bound, and emit frames only when the content changes. + /// `ScreenCaptureKit` is given it as the minimum frame interval; on Windows it is the minimum + /// update interval of the capture session (which needs Windows 11 22H2, and on an older + /// Windows the capture is held to 60 frames a second whatever this is). On Windows it is also + /// the rate declared to the encoder, which sets the bitrate (see [`segment::h264_bitrate`]), + /// the key frame spacing and the duration given to the last frame; with no cap that is + /// [`segment::UNCAPPED_NOMINAL_FPS`]. On a Mac the encoder never sees it. pub fps: u32, /// Whether to capture at native pixel resolution rather than logical points. /// @@ -171,16 +174,6 @@ pub struct CaptureConfig { pub codec: Codec, } -impl Default for CaptureConfig { - fn default() -> Self { - Self { - fps: 30, - retina: false, - codec: Codec::default(), - } - } -} - /// A display that can be captured, as the OS described it at one moment. /// /// A descriptor is a snapshot. [`CaptureBackend::active_displays`] returns a fresh list each time diff --git a/komorebi-recorder/src/capture/sck.rs b/komorebi-recorder/src/capture/sck.rs index 1380c7ed..b20e39c7 100644 --- a/komorebi-recorder/src/capture/sck.rs +++ b/komorebi-recorder/src/capture/sck.rs @@ -16,10 +16,10 @@ //! //! 1. [`DisplayCapture::start`] finds the display among `ScreenCaptureKit`'s shareable content, //! builds a filter that leaves out this process and any other `komorebi-recorder` process, and -//! configures the stream: the size, `fps` as the minimum frame interval, BGRA pixels, the cursor -//! shown. It creates the `AVAssetWriter` (an MP4 cut into movie fragments every second, so a -//! killed recorder leaves a file that plays up to the last fragment) and the frame log with its -//! header row, and starts the stream. +//! configures the stream: the size, `fps` as the minimum frame interval (none when it is `0`), +//! BGRA pixels, the cursor shown. It creates the `AVAssetWriter` (an MP4 cut into movie +//! fragments every second, so a killed recorder leaves a file that plays up to the last +//! fragment) and the frame log with its header row, and starts the stream. //! 2. Each frame arrives on a serial dispatch queue. A callback that carries no new content (the //! frame status is idle, blank or suspended) is counted in //! [`CaptureStats::idle_callbacks`] and dropped. A real frame starts the writer's session at its @@ -765,7 +765,13 @@ impl DisplayCapture { let stream_config = SCStreamConfiguration::new(); stream_config.setWidth(width_px); stream_config.setHeight(height_px); - stream_config.setMinimumFrameInterval(CMTime::new(1, config.fps.max(1) as i32)); + // zero is no minimum: frames as fast as the display refreshes + let minimum_interval = if config.fps == 0 { + CMTime::new(0, 1) + } else { + CMTime::new(1, config.fps as i32) + }; + stream_config.setMinimumFrameInterval(minimum_interval); stream_config.setPixelFormat(PIXEL_FORMAT_BGRA); stream_config.setShowsCursor(true); stream_config.setQueueDepth(6); diff --git a/komorebi-recorder/src/capture/segment.rs b/komorebi-recorder/src/capture/segment.rs index fd18990a..bd5296c6 100644 --- a/komorebi-recorder/src/capture/segment.rs +++ b/komorebi-recorder/src/capture/segment.rs @@ -121,6 +121,49 @@ pub fn h264_size(width: usize, height: usize) -> (usize, usize) { (even(width * scale), even(height * scale)) } +/// The frame rate declared to the encoder when the capture is not capped (`fps` of `0`). +/// +/// Frames are stamped with the times they arrived, so this only sets the bitrate, the key frame +/// spacing and the duration given to the last frame. +pub const UNCAPPED_NOMINAL_FPS: u32 = 60; + +/// Returns the frame rate to declare to the encoder for a capture configured with `fps`. +/// +/// # Example +/// +/// ```rust +/// use komorebi_recorder::capture::segment::{nominal_fps, UNCAPPED_NOMINAL_FPS}; +/// +/// assert_eq!(nominal_fps(24), 24); +/// assert_eq!(nominal_fps(0), UNCAPPED_NOMINAL_FPS); +/// ``` +#[must_use] +pub fn nominal_fps(fps: u32) -> u32 { + if fps == 0 { UNCAPPED_NOMINAL_FPS } else { fps } +} + +/// Returns the shortest time between frames to ask the Windows capture for, in 100 ns units. +/// +/// With a cap that is a frame at `fps`; with none (`fps` of `0`) it is a millisecond, which is +/// shorter than the refresh of any display. +/// +/// # Example +/// +/// ```rust +/// use komorebi_recorder::capture::segment::min_update_interval_100ns; +/// +/// assert_eq!(min_update_interval_100ns(30), 333_333); +/// assert_eq!(min_update_interval_100ns(0), 10_000); +/// ``` +#[must_use] +pub fn min_update_interval_100ns(fps: u32) -> i64 { + if fps == 0 { + 10_000 + } else { + 10_000_000 / i64::from(fps) + } +} + /// Returns the bitrate to encode a screen of this size at, in bits per second. /// /// Screen content is mostly flat and changes in bursts, so this is about a tenth of a bit per pixel diff --git a/komorebi-recorder/src/capture/wgc.rs b/komorebi-recorder/src/capture/wgc.rs index d2530186..184828ed 100644 --- a/komorebi-recorder/src/capture/wgc.rs +++ b/komorebi-recorder/src/capture/wgc.rs @@ -71,6 +71,8 @@ use super::segment::even; use super::segment::h264_bitrate; use super::segment::h264_size; use super::segment::keepalive_due; +use super::segment::min_update_interval_100ns; +use super::segment::nominal_fps; use super::video_file_name; use crate::clock; use crate::platform::media::pack; @@ -80,6 +82,7 @@ use parking_lot::Mutex; use std::path::Path; use std::sync::Arc; use std::sync::atomic::Ordering; +use windows::Foundation::TimeSpan; use windows::Foundation::TypedEventHandler; use windows::Graphics::Capture::Direct3D11CaptureFramePool; use windows::Graphics::Capture::GraphicsCaptureItem; @@ -401,6 +404,9 @@ struct Writer { /// How long one frame at the nominal rate lasts, in 100 ns units: what the last frame of a /// still display is given. frame_100ns: i64, + /// The latest frame, not yet given to the encoder: how long it lasts is the time until the one + /// after it, which is not known until that one arrives. + pending: Option<(IMFMediaBuffer, i64)>, } fn input_type(width: u32, height: u32, fps: u32) -> eyre::Result { @@ -439,7 +445,7 @@ fn output_type(width: u32, height: u32, fps: u32) -> eyre::Result impl Writer { fn create(path: &Path, gpu: &Gpu, width: u32, height: u32, fps: u32) -> eyre::Result { - let fps = fps.max(1); + let fps = nominal_fps(fps); let wide: Vec = path .to_string_lossy() .encode_utf16() @@ -481,6 +487,7 @@ impl Writer { sink, stream, frame_100ns: i64::from(10_000_000 / fps), + pending: None, }) } } @@ -497,14 +504,28 @@ impl Writer { } } - fn write(&self, buffer: &IMFMediaBuffer, time: i64) -> eyre::Result<()> { - let sample = Self::sample(buffer, time, self.frame_100ns)?; + /// Queues `buffer` to be shown from `time`, and hands the encoder the frame before it, which + /// is now known to have lasted until `time`. + fn write(&mut self, buffer: &IMFMediaBuffer, time: i64) -> eyre::Result<()> { + if let Some((previous, started)) = self.pending.replace((buffer.clone(), time)) { + self.submit(&previous, started, (time - started).max(1))?; + } + Ok(()) + } + + fn submit(&self, buffer: &IMFMediaBuffer, time: i64, duration: i64) -> eyre::Result<()> { + let sample = Self::sample(buffer, time, duration)?; // SAFETY: the writer is writing (`create` began it) and the sample is complete unsafe { self.sink.WriteSample(self.stream, &sample) } .wrap_err("the encoder rejected a frame") } - fn finish(&self) -> eyre::Result<()> { + /// Gives the encoder the last frame, for the length of one frame at the nominal rate, and + /// closes the file. + fn finish(&mut self) -> eyre::Result<()> { + if let Some((last, started)) = self.pending.take() { + self.submit(&last, started, self.frame_100ns)?; + } // SAFETY: the writer is writing; this waits for what is queued to be encoded and the file // to be closed unsafe { self.sink.Finalize() }.wrap_err("could not finish the video file") @@ -816,6 +837,13 @@ impl DisplayCapture { // best effort: each of these needs a newer Windows than the capture itself let _ = session.SetIsBorderRequired(false); let _ = session.SetIsCursorCaptureEnabled(true); + // Without this Windows holds the capture to 60 frames a second, so a 120 Hz display is + // recorded at half its rate and a one-refresh flash can fall between two frames. The + // interval is the cap asked for, or (with none) a millisecond, which is shorter than any + // refresh. It needs Windows 11 22H2, and older ones keep their 60 frames a second. + let _ = session.SetMinUpdateInterval(TimeSpan { + Duration: min_update_interval_100ns(config.fps), + }); session .StartCapture() .wrap_err("Windows refused to start the capture")?; diff --git a/komorebi-recorder/src/cli/record.rs b/komorebi-recorder/src/cli/record.rs index 85cd5c2a..dd6d8d1e 100644 --- a/komorebi-recorder/src/cli/record.rs +++ b/komorebi-recorder/src/cli/record.rs @@ -36,8 +36,8 @@ use std::time::Duration; /// The options of a recording, shared by `record` and `gui`. /// -/// The default is a recording with no flags given: every display, 30 frames per second at logical -/// resolution, komorebi and the OS trace included, no time limit. +/// The default is a recording with no flags given: every display, frames as fast as the display +/// refreshes at logical resolution, komorebi and the OS trace included, no time limit. #[derive(Args, Clone, Debug)] pub struct RecordArgs { /// Session directory to create [default: in komorebi's data directory, under recordings/, @@ -50,8 +50,9 @@ pub struct RecordArgs { /// Display id to record (repeatable; see `displays`) [default: every display] #[arg(long = "display", value_name = "ID")] pub displays: Vec, - /// Maximum video frame rate (frames are only produced when the screen changes) - #[arg(long, default_value_t = 30)] + /// Maximum video frame rate, 0 for as fast as the display refreshes (frames are only produced + /// when the screen changes) + #[arg(long, default_value_t = 0)] pub fps: u32, /// Record at native pixel resolution instead of logical points #[arg(long)] @@ -114,7 +115,7 @@ impl Default for RecordArgs { out: None, duration: None, displays: Vec::new(), - fps: 30, + fps: 0, retina: false, codec: Codec::default(), no_komorebi: false, @@ -159,7 +160,7 @@ impl RecordArgs { /// assert_eq!(options.duration, Some(Duration::from_millis(2500))); /// assert_eq!(options.displays, [1]); /// assert!(!options.oslog.enabled); - /// assert_eq!(options.capture.fps, 30); + /// assert_eq!(options.capture.fps, 0); /// ``` #[must_use] pub fn to_options(&self) -> RecordOptions { diff --git a/komorebi-recorder/src/testkit.rs b/komorebi-recorder/src/testkit.rs index 6ae45121..043dbbe6 100644 --- a/komorebi-recorder/src/testkit.rs +++ b/komorebi-recorder/src/testkit.rs @@ -679,8 +679,9 @@ pub fn komorebi_mac_src() -> PathBuf { /// checkout, and this build indexed it (an empty catalog means it was built without one). /// /// A machine that has neither (Windows, or a Mac outside `nix develop`) has nothing for those -/// tests to check, which is not a failure: they use [`skip_without_komorebi_mac_source`] to say so -/// and stop, as the tests that need `ffmpeg` do. +/// tests to check, which is not a failure: they use +/// [`skip_without_komorebi_mac_source!`](crate::skip_without_komorebi_mac_source) to say so and +/// stop, as the tests that need `ffmpeg` do. #[must_use] pub fn komorebi_mac_source_available() -> bool { std::env::var_os("KOMOREBI_MAC_SRC").is_some_and(|path| !path.is_empty()) diff --git a/komorebi-recorder/src/viewer/rules.rs b/komorebi-recorder/src/viewer/rules.rs index f5121079..3ed9eb15 100644 --- a/komorebi-recorder/src/viewer/rules.rs +++ b/komorebi-recorder/src/viewer/rules.rs @@ -118,7 +118,7 @@ impl Open { /// Sends the rule the form shows, in the background, with the tools and the repository `make` /// gives (it runs on the thread: finding `gh` runs it). /// - /// Does nothing unless the form is open, at [`Stage::Review`], and has a rule to send. The + /// Does nothing unless the form is open, at `Stage::Review`, and has a rule to send. The /// window sends to the real repository with [`Open::send_rule`]; this is its seam for the tests. pub fn start_sending(&mut self, make: impl FnOnce() -> (Tools, Upstream) + Send + 'static) { let Some(form) = self.rule.as_mut() else {