From 3e30d81bc544ce1dc350665518d02426a9c3bd41 Mon Sep 17 00:00:00 2001 From: LGUG2Z Date: Tue, 27 Oct 2026 06:14:44 -0700 Subject: [PATCH] feat(av): decode the recorded video Reading the video back: AVFoundation on macOS, the Media Foundation source reader on Windows, a clear error elsewhere. A request for the frames between two times is answered with the frame that was on screen at the start, even when the decoder had to seek to an earlier key frame. `selftest` records a rendered on-screen clock and compares the clock the video shows with the time the session predicts. --- Cargo.lock | 1 + docs/PLATFORM.md | 6 + docs/usage/recording-a-bug-repro.md | 5 + komorebi-recorder/Cargo.toml | 8 + komorebi-recorder/src/av/mf.rs | 368 ++++++++ komorebi-recorder/src/av/mod.rs | 329 +++++++ komorebi-recorder/src/av/reader.rs | 489 ++++++++++ komorebi-recorder/src/av/unsupported.rs | 78 ++ komorebi-recorder/src/cli/mod.rs | 21 + komorebi-recorder/src/lib.rs | 3 + komorebi-recorder/src/platform/win/capture.rs | 8 +- komorebi-recorder/src/selftest.rs | 844 ++++++++++++++++++ 12 files changed, 2156 insertions(+), 4 deletions(-) create mode 100644 komorebi-recorder/src/av/mf.rs create mode 100644 komorebi-recorder/src/av/mod.rs create mode 100644 komorebi-recorder/src/av/reader.rs create mode 100644 komorebi-recorder/src/av/unsupported.rs create mode 100644 komorebi-recorder/src/selftest.rs diff --git a/Cargo.lock b/Cargo.lock index a07ef11a..8476cc76 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1914,6 +1914,7 @@ dependencies = [ "objc2-core-foundation", "objc2-core-graphics", "objc2-core-media", + "objc2-core-video", "objc2-foundation 0.3.2", "objc2-screen-capture-kit", "parking_lot", diff --git a/docs/PLATFORM.md b/docs/PLATFORM.md index a21c8ce0..cb6a7331 100644 --- a/docs/PLATFORM.md +++ b/docs/PLATFORM.md @@ -102,6 +102,12 @@ the locked file. The wire protocol is the same on both. The recorder's own binary does not depend on either komorebi's crates. +### Decoding the recorded video + +macOS decodes with AVFoundation's `AVAssetReader`, Windows with Media Foundation's Source Reader, and the `unsupported` +host with nothing: it says so when asked to open a file. Both real hosts are asked for BGRA frames at a chosen size, so +that a contact sheet never decodes more pixels than it shows. + ### The program's own windows The recorder window and the viewer must not be in the video they are recording, and komorebi must not tile them. diff --git a/docs/usage/recording-a-bug-repro.md b/docs/usage/recording-a-bug-repro.md index f64566ba..199e1467 100644 --- a/docs/usage/recording-a-bug-repro.md +++ b/docs/usage/recording-a-bug-repro.md @@ -54,6 +54,8 @@ The recorder window keeps itself out of the video (Windows 10 version 2004 or la - **komorebi's log level.** The recorder never changes komorebi's log level or lifecycle; it only warns. For a useful repro, start komorebi with `komorebi --log-level trace` (or `RUST_LOG=komorebi=trace`) first. At the default `info` level most of what the window manager _decided_ is not in the log. +- **`ffmpeg`** is only needed for `frames` and `selftest`. It is never used to capture. The `nix develop` shell provides + it; otherwise `brew install ffmpeg`. The recorder window is borderless and untitled on purpose: komorebi does not manage windows without a title, so opening it does not move your layout around while you are trying to reproduce a layout bug. It is also excluded from the video, as is any other `komorebi-recorder` process. @@ -167,6 +169,9 @@ anchors sampled every 5 seconds during the recording and at every sleep, so cloc the event, so the stamp is when the handler **finished**; the `processing event` line in `komorebi.log` is when it began. +`komorebi-recorder selftest` checks this on your machine: it records a rendered on-screen clock and compares the clock +visible in the video with the time the session predicts. The result is stored in `session.json` under `calibration`. + ## Things to know - **What can still be missing.** Events from a `log stream` during the moment it was dead and being restarted; komorebi diff --git a/komorebi-recorder/Cargo.toml b/komorebi-recorder/Cargo.toml index e5db3af8..4aae5819 100644 --- a/komorebi-recorder/Cargo.toml +++ b/komorebi-recorder/Cargo.toml @@ -68,6 +68,14 @@ objc2-core-media = { version = "0.3", default-features = false, features = [ "CMTime", "CMTimeRange", ] } +objc2-core-video = { version = "0.3", default-features = false, features = [ + "std", + "CVBase", + "CVBuffer", + "CVImageBuffer", + "CVPixelBuffer", + "CVReturn", +] } objc2-foundation = { version = "0.3", default-features = false, features = [ "std", "NSArray", diff --git a/komorebi-recorder/src/av/mf.rs b/komorebi-recorder/src/av/mf.rs new file mode 100644 index 00000000..85fc1c3a --- /dev/null +++ b/komorebi-recorder/src/av/mf.rs @@ -0,0 +1,368 @@ +//! Decoding recorded video on Windows with Media Foundation's Source Reader. +//! +//! The reader is asked for BGRA frames at the size wanted (its video processor converts and +//! scales, so a contact sheet never decodes more pixels than it shows), seeks to the key frame +//! before the time asked for, and decodes forward; [`super::Selector`] says which of the decoded +//! frames are the answer. Media Foundation objects belong to the thread that made them as far as +//! this code is concerned, so a decode is started and read on one thread, as the Mac's are. +//! +//! An [`Asset`] keeps the path and what was read from the file when it was opened (size, duration, +//! whether it is HEVC), not an open reader: every [`Asset::start`] opens a new Source Reader, so a +//! file that changes after it was opened is read as it is then. +//! +//! The recorder writes H.264 on Windows, which Media Foundation decodes everywhere. An HEVC file (a +//! recording made on a Mac) needs a decoder Windows does not always have, and the error from +//! [`Asset::start`] tells the user to install the HEVC Video Extensions or to record with H.264. +//! +//! # Tests +//! +//! The rule for which decoded frames answer a request is `Selector`, and its tests run on every +//! host. Nothing in this module runs without Windows and a recorded video: `doctor --probe decode` +//! measures first-frame time, decode speed and seeking there, and the checkpoint W3 in +//! `docs/dev/windows-checkpoints.md` is its first run. + +use super::Frame; +use super::Scale; +use super::Selector; +use crate::platform::media::pack; +use crate::platform::media::unpack; +use color_eyre::eyre; +use color_eyre::eyre::WrapErr; +use std::collections::VecDeque; +use std::path::Path; +use windows::Win32::Media::MediaFoundation::IMF2DBuffer; +use windows::Win32::Media::MediaFoundation::IMFAttributes; +use windows::Win32::Media::MediaFoundation::IMFSample; +use windows::Win32::Media::MediaFoundation::IMFSourceReader; +use windows::Win32::Media::MediaFoundation::MF_MT_DEFAULT_STRIDE; +use windows::Win32::Media::MediaFoundation::MF_MT_FRAME_SIZE; +use windows::Win32::Media::MediaFoundation::MF_MT_INTERLACE_MODE; +use windows::Win32::Media::MediaFoundation::MF_MT_MAJOR_TYPE; +use windows::Win32::Media::MediaFoundation::MF_MT_SUBTYPE; +use windows::Win32::Media::MediaFoundation::MF_PD_DURATION; +use windows::Win32::Media::MediaFoundation::MF_SOURCE_READER_ALL_STREAMS; +use windows::Win32::Media::MediaFoundation::MF_SOURCE_READER_ENABLE_VIDEO_PROCESSING; +use windows::Win32::Media::MediaFoundation::MF_SOURCE_READER_FIRST_VIDEO_STREAM; +use windows::Win32::Media::MediaFoundation::MF_SOURCE_READER_MEDIASOURCE; +use windows::Win32::Media::MediaFoundation::MF_SOURCE_READERF_ENDOFSTREAM; +use windows::Win32::Media::MediaFoundation::MFCreateAttributes; +use windows::Win32::Media::MediaFoundation::MFCreateMediaType; +use windows::Win32::Media::MediaFoundation::MFCreateSourceReaderFromURL; +use windows::Win32::Media::MediaFoundation::MFMediaType_Video; +use windows::Win32::Media::MediaFoundation::MFVideoFormat_HEVC; +use windows::Win32::Media::MediaFoundation::MFVideoFormat_RGB32; +use windows::Win32::Media::MediaFoundation::MFVideoInterlace_Progressive; +use windows::Win32::System::Com::StructuredStorage::PROPVARIANT; +use windows::Win32::System::Com::StructuredStorage::PropVariantToUInt64; +use windows::core::GUID; +use windows::core::Interface; +use windows::core::PCWSTR; + +/// The first video stream, as the Source Reader numbers streams. +const VIDEO: u32 = MF_SOURCE_READER_FIRST_VIDEO_STREAM.0 as u32; + +/// Media Foundation counts in hundred-nanosecond units. +const UNITS_PER_SECOND: f64 = 10_000_000.0; + +fn open_reader(path: &Path) -> eyre::Result { + crate::platform::media::start()?; + if !path.is_file() { + eyre::bail!("{} does not exist", path.display()); + } + + let wide: Vec = path + .to_string_lossy() + .encode_utf16() + .chain(Some(0)) + .collect(); + // SAFETY: attribute setters and a creation call with the arguments they document; `wide` + // outlives the call that reads it + unsafe { + let mut attributes: Option = None; + MFCreateAttributes(&raw mut attributes, 1)?; + let attributes = + attributes.ok_or_else(|| eyre::Report::msg("no Media Foundation attributes"))?; + // the video processor converts the decoded picture to the format asked for and resizes it + attributes.SetUINT32(&MF_SOURCE_READER_ENABLE_VIDEO_PROCESSING, 1)?; + + let reader = MFCreateSourceReaderFromURL(PCWSTR(wide.as_ptr()), &attributes) + .wrap_err_with(|| format!("Windows cannot open {} as a video", path.display()))?; + reader.SetStreamSelection(MF_SOURCE_READER_ALL_STREAMS.0 as u32, false)?; + reader.SetStreamSelection(VIDEO, true)?; + Ok(reader) + } +} + +/// An opened video file: its path, size and duration. +/// +/// Opening reads those facts and decodes nothing. Each [`Asset::start`] opens a new Source +/// Reader on the file and starts where it is told to. +pub struct Asset { + path: std::path::PathBuf, + width: u32, + height: u32, + duration_s: f64, + /// Whether the video is HEVC, which needs a decoder Windows does not always have. + hevc: bool, +} + +impl Asset { + /// Opens a video file. + /// + /// # Errors + /// + /// Returns an error when: + /// + /// - Windows Runtime or Media Foundation cannot be started; + /// - `path` is not a file that exists (` does not exist`); + /// - Windows cannot open the file as a video (`Windows cannot open as a video`); + /// - the file has no video track (` has no video track`), or its frame size cannot be + /// read. + pub fn open(path: &Path) -> eyre::Result { + let reader = open_reader(path)?; + + // SAFETY: property reads on a live reader + unsafe { + let native = reader + .GetNativeMediaType(VIDEO, 0) + .wrap_err_with(|| format!("{} has no video track", path.display()))?; + let (width, height) = unpack(native.GetUINT64(&MF_MT_FRAME_SIZE)?); + let hevc = native + .GetGUID(&MF_MT_SUBTYPE) + .is_ok_and(|subtype| subtype == MFVideoFormat_HEVC); + + let duration = reader + .GetPresentationAttribute(MF_SOURCE_READER_MEDIASOURCE.0 as u32, &MF_PD_DURATION) + .ok() + .and_then(|value| PropVariantToUInt64(&value).ok()) + .map_or(0.0, |units| units as f64 / UNITS_PER_SECOND); + + Ok(Self { + path: path.to_path_buf(), + width, + height, + duration_s: duration, + hevc, + }) + } + } + + /// Returns the size the video was recorded at, as `(width, height)` in pixels. + #[must_use] + pub fn size(&self) -> (u32, u32) { + (self.width, self.height) + } + + /// Returns the duration Media Foundation reports for the file, in seconds, or `0.0` if it + /// reports none (some truncated files). + #[must_use] + pub fn duration_s(&self) -> f64 { + self.duration_s + } + + /// Begins decoding at `from_s` seconds into the file, up to `to_s` if given (else the end). + /// + /// The first frame returned is the one on screen at `from_s`, which can be earlier than it. A + /// negative `from_s` is `0`. The end is `to_s`, or the file's + /// [`duration_s`](Asset::duration_s) if it is `None`, and is never before `from_s`: a file + /// whose duration is unknown (`0.0`) therefore decodes little more than the frame on screen at + /// `from_s` unless `to_s` is given. A frame at exactly the end is returned, and none after it. + /// + /// The Source Reader is asked for BGRA frames of the size `scale` asks for (see [`Scale`]). + /// When `from_s` is not `0` it seeks to the key frame at or before it, and the frames from + /// there up to `from_s` are decoded and passed over: a seek costs the frames between a key + /// frame and the time asked for. + /// + /// # Errors + /// + /// Returns an error when: + /// + /// - the file cannot be opened again (see [`Asset::open`]); + /// - the video is HEVC and Windows has no HEVC decoder (the message tells the user to install + /// the HEVC Video Extensions or to record with `--codec h264`); + /// - Windows cannot decode the video to pictures of that size and format; + /// - the seek to `from_s` fails (`could not seek in the video`). + pub fn start(&self, from_s: f64, to_s: Option, scale: Scale) -> eyre::Result { + let from = from_s.max(0.0); + let end = to_s.unwrap_or(self.duration_s).max(from); + let (width, height) = match scale { + Scale::Native => (self.width, self.height), + Scale::Width(wide) => { + let wide = wide.clamp(2, self.width.max(2)); + let high = (u64::from(wide) * u64::from(self.height) + / u64::from(self.width.max(1))) + .max(2); + // codecs like even sizes + (wide & !1, (high as u32) & !1) + } + }; + + let reader = open_reader(&self.path)?; + // SAFETY: setters on a media type just created and a reader just opened + unsafe { + let wanted = MFCreateMediaType()?; + wanted.SetGUID(&MF_MT_MAJOR_TYPE, &MFMediaType_Video)?; + wanted.SetGUID(&MF_MT_SUBTYPE, &MFVideoFormat_RGB32)?; + wanted.SetUINT32(&MF_MT_INTERLACE_MODE, MFVideoInterlace_Progressive.0 as u32)?; + wanted.SetUINT64(&MF_MT_FRAME_SIZE, pack(width, height))?; + // a positive stride is a top-down picture; Media Foundation's default for RGB is the + // other way up + wanted.SetUINT32(&MF_MT_DEFAULT_STRIDE, width * 4)?; + + if let Err(error) = reader.SetCurrentMediaType(VIDEO, None, &wanted) { + if self.hevc { + eyre::bail!( + "this video is HEVC and Windows has no HEVC decoder installed: install \ + the \"HEVC Video Extensions\" from the Microsoft Store, or record with \ + --codec h264 ({error})" + ); + } + return Err(error).wrap_err("Windows cannot decode this video to pictures"); + } + + if from > 0.0 { + // to the key frame at or before it; the frames up to `from` are decoded and passed + // over by the selector + reader + .SetCurrentPosition( + &GUID::zeroed(), + &PROPVARIANT::from((from * UNITS_PER_SECOND) as i64), + ) + .wrap_err("could not seek in the video")?; + } + } + + Ok(Reader { + reader, + selector: Selector::new(from, end), + ready: VecDeque::new(), + ended: false, + }) + } +} + +/// A started decode. Dropping it cancels the decode. +/// +/// Made only by [`Asset::start`]. It moves forward only: to look at another time, start another +/// decode. The frames it returns are the ones a `Selector` picks from what the Source Reader +/// decodes, so they begin with the frame on screen at the start of the range. +pub struct Reader { + reader: IMFSourceReader, + selector: Selector, + ready: VecDeque, + ended: bool, +} + +/// One decoded sample as a tightly packed BGRA frame. +fn frame_of(reader: &IMFSourceReader, sample: &IMFSample, time: i64) -> eyre::Result { + // SAFETY: the sample is live; the buffer is locked for exactly the copy and unlocked after + unsafe { + let current = reader.GetCurrentMediaType(VIDEO)?; + let (width, height) = unpack(current.GetUINT64(&MF_MT_FRAME_SIZE)?); + let (width, height) = (width as usize, height as usize); + + let buffer = sample.ConvertToContiguousBuffer()?; + let mut bgra = vec![0_u8; width * height * 4]; + + if let Ok(flat) = buffer.cast::() { + let (mut first, mut pitch) = (std::ptr::null_mut::(), 0_i32); + flat.Lock2D(&raw mut first, &raw mut pitch)?; + for row in 0..height { + // the pitch is negative for a picture stored bottom-up, and `first` is then the + // top row all the same + let source = first.offset(row as isize * pitch as isize); + std::ptr::copy_nonoverlapping( + source, + bgra.as_mut_ptr().add(row * width * 4), + width * 4, + ); + } + flat.Unlock2D()?; + } else { + let (mut data, mut length) = (std::ptr::null_mut::(), 0_u32); + buffer.Lock(&raw mut data, None, Some(&raw mut length))?; + let usable = (length as usize).min(bgra.len()); + std::ptr::copy_nonoverlapping(data, bgra.as_mut_ptr(), usable); + buffer.Unlock()?; + } + + // RGB32 leaves the fourth byte of each pixel undefined, and what draws the picture reads + // it as transparency + for pixel in bgra.as_chunks_mut::<4>().0 { + pixel[3] = 255; + } + + Ok(Frame { + pts_s: time as f64 / UNITS_PER_SECOND, + width: width as u32, + height: height as u32, + bgra, + }) + } +} + +impl Reader { + /// The next decoded frame of the file, or `None` at its end. + fn decode(&mut self) -> eyre::Result> { + loop { + let (mut flags, mut time) = (0_u32, 0_i64); + let mut sample: Option = None; + // SAFETY: the reader was set up by `start`; the out pointers are valid for the call + unsafe { + self.reader.ReadSample( + VIDEO, + 0, + None, + Some(&raw mut flags), + Some(&raw mut time), + Some(&raw mut sample), + ) + } + .wrap_err("could not read a frame")?; + + if flags & MF_SOURCE_READERF_ENDOFSTREAM.0 as u32 != 0 { + return Ok(None); + } + + // a tick with no picture, in a gap of the file: nothing yet + let Some(sample) = sample else { + continue; + }; + return frame_of(&self.reader, &sample, time).map(Some); + } + } + + /// Returns the next frame, or `None` at the end of the range. + /// + /// Frames come in order. Each is BGRA with the alpha byte set to opaque, because Media + /// Foundation leaves it undefined and what draws the picture reads it as transparency. + /// + /// # Errors + /// + /// Returns an error when the Source Reader fails to read a sample (`could not read a frame`), + /// or a decoded sample cannot be read as a picture. + pub fn next_frame(&mut self) -> eyre::Result> { + loop { + if let Some(frame) = self.ready.pop_front() { + return Ok(Some(frame)); + } + if self.selector.done() { + return Ok(None); + } + + let more = if self.ended { + Vec::new() + } else if let Some(frame) = self.decode()? { + self.selector.push(frame) + } else { + self.ended = true; + self.selector.finish() + }; + + if self.ended && more.is_empty() { + return Ok(None); + } + self.ready.extend(more); + } + } +} diff --git a/komorebi-recorder/src/av/mod.rs b/komorebi-recorder/src/av/mod.rs new file mode 100644 index 00000000..729c38ae --- /dev/null +++ b/komorebi-recorder/src/av/mod.rs @@ -0,0 +1,329 @@ +//! Decoding the recorded video. +//! +//! The recorder writes each display's video as a fragmented MP4 (see [`crate::capture`]). Reading +//! it back means asking the OS for the frames between two times, decoded to BGRA at a size of the +//! caller's choosing (the decoder scales, so a contact sheet never decodes more pixels than it +//! shows). This is the host axis of reading a session: the types below are the same everywhere and +//! the decoder behind them is the host's. +//! +//! | host | decoder | module | +//! |---|---|---| +//! | macOS | `AVFoundation`'s `AVAssetReader` | `reader` | +//! | Windows | Media Foundation's Source Reader | `mf` | +//! | any other | none: [`Asset::open`] fails | `unsupported` | +//! +//! The decoder modules are private. [`Asset`] and [`Reader`] are re-exports of the host's, and +//! [`supported`] says whether there is one. A host with no decoder can still read everything else +//! in a session. +//! +//! # Using it +//! +//! [`Asset::open`] a file, [`Asset::start`] a decode of a range of it at a [`Scale`], and call +//! [`Reader::next_frame`] until it returns `None`. The first frame is the one that was on screen at +//! the start of the range, and a recorded display only has frames where the screen changed, so a +//! range can hold few frames or one. +//! +//! ```rust,no_run +//! use komorebi_recorder::av::{Asset, Scale}; +//! use std::path::Path; +//! +//! let asset = Asset::open(Path::new("video-1.mp4"))?; +//! let (width, height) = asset.size(); +//! println!("{width}x{height}, {:.1} s", asset.duration_s()); +//! +//! // The frames of the second second of the file, decoded 320 pixels wide. +//! let mut reader = asset.start(1.0, Some(2.0), Scale::Width(320))?; +//! while let Some(frame) = reader.next_frame()? { +//! assert_eq!(frame.bgra.len(), frame.width as usize * frame.height as usize * 4); +//! println!("frame at {:.3} s", frame.pts_s); +//! } +//! # Ok::<(), Box>(()) +//! ``` +//! +//! The example needs a recorded video, so it is compiled and not run. What it does is covered on +//! macOS by the unit tests in `av::reader` (they build a video whose frames number themselves and +//! need `ffmpeg`), and on every host by the tests of the frame selection rule (see the private +//! `Selector`). Nothing exercises the Windows decoder without a Windows machine, where +//! `doctor --probe decode` measures it. +//! +//! # Why it is shaped this way +//! +//! `AVFoundation`'s objects are not `Send` and some calls raise an Objective-C exception instead of +//! returning an error when used in the wrong order, which would abort the process. The types here +//! make the wrong order impossible to write: the only way to a [`Reader`] is +//! [`Asset::start`], which configures everything before reading begins. A Media Foundation object +//! is likewise used only on the thread that made it, so on Windows too a decode is started and read +//! on one thread. +//! +//! # What this module does not do +//! +//! It does not decide which frames a recording means: choosing the moments to look at, comparing +//! frames and drawing contact sheets are done by `analysis` and `playback`, on top of this. It +//! decodes video only. It does not seek within a started decode: start another [`Reader`]. + +/// Returns `true` if and only if this host can decode the recorded video. +/// +/// That is macOS and Windows. Elsewhere [`Asset::open`] always fails. +/// +/// # Example +/// +/// ```rust +/// assert_eq!( +/// komorebi_recorder::av::supported(), +/// cfg!(any(target_os = "macos", windows)) +/// ); +/// ``` +#[must_use] +pub const fn supported() -> bool { + cfg!(any(target_os = "macos", windows)) +} + +#[cfg(windows)] +mod mf; +#[cfg(target_os = "macos")] +mod reader; +#[cfg(not(any(target_os = "macos", windows)))] +mod unsupported; + +#[cfg(windows)] +pub use mf::Asset; +#[cfg(windows)] +pub use mf::Reader; +#[cfg(target_os = "macos")] +pub use reader::Asset; +#[cfg(target_os = "macos")] +pub use reader::Reader; +#[cfg(not(any(target_os = "macos", windows)))] +pub use unsupported::Asset; +#[cfg(not(any(target_os = "macos", windows)))] +pub use unsupported::Reader; + +/// The size to decode frames at. +/// +/// The decoder does the scaling, so a request for small frames decodes fewer pixels than the +/// recording has. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Scale { + /// The size the video was recorded at ([`Asset::size`]). + Native, + /// This wide, keeping the aspect ratio. The width is limited to between 2 and the recorded + /// width (a frame is never scaled up), and the height follows from the aspect ratio. Both are + /// rounded down to an even number, as video codecs want, and are never less than 2. + Width(u32), +} + +/// One decoded frame, tightly packed BGRA (no row padding). +/// +/// The `bgra` field is `width * height * 4` bytes: the blue, green, red and alpha bytes of each +/// pixel, row by row from the top. +#[derive(Debug, Clone, PartialEq)] +pub struct Frame { + /// When the frame is shown, in seconds from the start of *this file*. The file's time zero is + /// its first frame, which was captured at [`DisplayResult::video_t0_mach_ns`]; the session's + /// video time of a file is offset from it by [`DisplayEntry::offset_s`]. + /// + /// [`DisplayResult::video_t0_mach_ns`]: crate::capture::DisplayResult::video_t0_mach_ns + /// [`DisplayEntry::offset_s`]: crate::session::DisplayEntry::offset_s + pub pts_s: f64, + /// The width of the picture in pixels. + pub width: u32, + /// The height of the picture in pixels. + pub height: u32, + /// The pixels: `width * height * 4` bytes, BGRA, top row first. Alpha is opaque on Windows, + /// where the decoder leaves it undefined. + pub bgra: Vec, +} + +/// Picks the frames a request for `from..=end` is answered with from the decoded frames of a file, +/// given in order. +/// +/// A decoder that seeks can only land on a key frame before the time asked for and decodes forward +/// from there, so the frames before `from` have to be told from the one that was on screen at +/// `from`, which is the last of them: it is the first frame the request gets, and it can be earlier +/// than `from`. Frames after `end` are not wanted and the first of them ends the request. +/// +/// The Windows decoder (`mf`) uses it, because Media Foundation's Source Reader seeks to the key +/// frame before the position and decodes forward. The macOS decoder (`reader`) does not: an +/// `AVAssetReader` given a time range starts at the frame on screen at its start. The rule is +/// host-independent, so its tests run on every host. +/// +/// # Protocol +/// +/// Create it with the request's `from` and `end` in seconds of the file's time, [`push`] each +/// decoded frame in order and collect what comes back, stop as soon as [`done`] is `true`, and if +/// the decoder runs out of frames first, collect [`finish`]. +/// +/// ```text +/// request 1.25..=2.0; the decoder lands on a key frame at 0.0 and decodes forward +/// +/// push(0.0) -> [] held: before the start, it may be the one on screen +/// push(0.5) -> [] held instead of 0.0 +/// push(1.0) -> [] held instead of 0.5 +/// push(1.5) -> [1.0, 1.5] the first frame at or after the start: the held 1.0 was on screen at +/// 1.25, so it is the request's first frame +/// push(2.0) -> [2.0] a frame exactly at the end is still wanted +/// push(2.5) -> [] later than the end: done() becomes true +/// ``` +/// +/// Edge cases, each covered by a unit test of this module: +/// +/// - a frame exactly at `from` is the first frame, and the held one before it is dropped; +/// - a request after the last frame of a still display is answered by [`finish`] with the last +/// frame before `from`, because nothing changed after it; +/// - a request before the first frame is answered from the first frame; +/// - an empty file answers with nothing. +/// +/// [`push`]: Selector::push +/// [`done`]: Selector::done +/// [`finish`]: Selector::finish +#[derive(Debug)] +#[cfg_attr(not(windows), allow(dead_code))] +pub(crate) struct Selector { + from_s: f64, + end_s: f64, + started: bool, + /// The most recent frame before `from`. + pending: Option, + done: bool, +} + +#[cfg_attr(not(windows), allow(dead_code))] +impl Selector { + /// Creates a selector for the frames shown from `from_s` to `end_s`, both in seconds of the + /// file's time. + pub(crate) fn new(from_s: f64, end_s: f64) -> Self { + Self { + from_s, + end_s, + started: false, + pending: None, + done: false, + } + } + + /// Returns `true` if and only if the request is answered in full: nothing more is wanted. + pub(crate) fn done(&self) -> bool { + self.done + } + + fn within(&mut self, frame: Frame) -> Vec { + if frame.pts_s > self.end_s { + self.done = true; + Vec::new() + } else { + vec![frame] + } + } + + /// Offers the next decoded frame and returns the frames that are ready to be handed out, in + /// order: none, one, or the held frame followed by this one. + /// + /// Returns nothing once the request is [done](Selector::done). + pub(crate) fn push(&mut self, frame: Frame) -> Vec { + if self.done { + return Vec::new(); + } + + if !self.started { + if frame.pts_s < self.from_s { + self.pending = Some(frame); + return Vec::new(); + } + + self.started = true; + // a frame exactly at `from` is the one on screen then; only when the first at or after + // it is later does the one before it still hold the screen + let before = self.pending.take().filter(|_| frame.pts_s > self.from_s); + let mut ready: Vec = before.into_iter().collect(); + ready.extend(self.within(frame)); + return ready; + } + + self.within(frame) + } + + /// Tells the selector that the file has no more frames, and returns what is left to hand out. + /// + /// If nothing later was found, the frame that was on screen at `from` is the answer (a still + /// display, where no frame follows). After a frame at or after `from` has been pushed, nothing + /// is left. + pub(crate) fn finish(&mut self) -> Vec { + self.done = true; + if self.started { + return Vec::new(); + } + + self.started = true; + self.pending.take().into_iter().collect() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn frame(pts_s: f64) -> Frame { + Frame { + pts_s, + width: 2, + height: 2, + bgra: vec![0; 16], + } + } + + fn times(frames: &[Frame]) -> Vec { + frames.iter().map(|frame| frame.pts_s).collect() + } + + fn run(from: f64, end: f64, decoded: &[f64]) -> Vec { + let mut selector = Selector::new(from, end); + let mut out = Vec::new(); + for pts in decoded { + out.extend(selector.push(frame(*pts))); + if selector.done() { + return times(&out); + } + } + out.extend(selector.finish()); + times(&out) + } + + #[test] + fn the_first_frame_is_the_one_that_was_on_screen_at_the_start() { + // decoding began at a key frame at 0.0; the request starts at 1.25 + assert_eq!( + run(1.25, 2.0, &[0.0, 0.5, 1.0, 1.5, 2.0, 2.5]), + [1.0, 1.5, 2.0] + ); + } + + #[test] + fn a_frame_exactly_at_the_start_is_the_first() { + assert_eq!(run(1.0, 1.5, &[0.0, 1.0, 1.5, 2.0]), [1.0, 1.5]); + } + + #[test] + fn nothing_after_the_end_is_returned_and_the_request_stops_there() { + let mut selector = Selector::new(0.0, 1.0); + assert_eq!(times(&selector.push(frame(0.5))), [0.5]); + assert_eq!(times(&selector.push(frame(1.5))), Vec::::new()); + assert!(selector.done()); + assert!(selector.push(frame(1.6)).is_empty()); + } + + #[test] + fn a_still_display_answers_with_the_last_frame_before_the_start() { + // the screen did not change after 1.0 (and the file ends): that is what was on screen at 5 + assert_eq!(run(5.0, 6.0, &[0.0, 1.0]), [1.0]); + } + + #[test] + fn a_request_before_the_first_frame_gets_the_first_frame() { + assert_eq!(run(0.0, 1.0, &[0.2, 0.6]), [0.2, 0.6]); + } + + #[test] + fn an_empty_file_answers_with_nothing() { + assert_eq!(run(0.0, 1.0, &[]), Vec::::new()); + } +} diff --git a/komorebi-recorder/src/av/reader.rs b/komorebi-recorder/src/av/reader.rs new file mode 100644 index 00000000..6a94a33e --- /dev/null +++ b/komorebi-recorder/src/av/reader.rs @@ -0,0 +1,489 @@ +//! Decoding recorded video on macOS with `AVFoundation`'s `AVAssetReader`. +//! +//! [`Asset`] is a file's first video track and the facts about it; [`Asset::start`] configures an +//! `AVAssetReader` completely (the time range, then an output asking for BGRA pixels at the size +//! wanted) before it calls `startReading`, which is the only order `AVFoundation` accepts without +//! raising an Objective-C exception. The decoder converts and scales, so a request for small frames +//! decodes fewer pixels than the recording has. The reader returns the frame that was on screen at +//! the start of the range as its first, so this module needs no frame selection (the +//! Windows decoder uses `Selector` for that). +//! +//! Measured on an Apple Silicon Mac with HEVC recordings of real displays, at time of writing +//! (2026-10-26): the first frame at a time arrives in 14 to 28 ms once warm, frames come at 250 to +//! 970 a second depending on the size, and the first open of a file takes about a second. The +//! ignored test `real_video_decode_speed` measures it on a recording named by `KOMOREBI_VIDEO`. +//! +//! The unit tests build a video whose frames are numbered (with `ffmpeg`; they skip +//! with a message without it) and read the number back out of the pixels, so seeking, ranges and +//! scaling are checked on what was decoded. + +use super::Frame; +use super::Scale; +use color_eyre::eyre; +use objc2::rc::Retained; +use objc2::runtime::AnyObject; +use objc2_av_foundation::AVAsset; +use objc2_av_foundation::AVAssetReader; +use objc2_av_foundation::AVAssetReaderOutput; +use objc2_av_foundation::AVAssetReaderStatus; +use objc2_av_foundation::AVAssetReaderTrackOutput; +use objc2_av_foundation::AVAssetTrack; +use objc2_av_foundation::AVMediaTypeVideo; +use objc2_core_media::CMTime; +use objc2_core_media::CMTimeRange; +use objc2_core_video::CVPixelBufferGetBaseAddress; +use objc2_core_video::CVPixelBufferGetBytesPerRow; +use objc2_core_video::CVPixelBufferGetHeight; +use objc2_core_video::CVPixelBufferGetWidth; +use objc2_core_video::CVPixelBufferLockBaseAddress; +use objc2_core_video::CVPixelBufferLockFlags; +use objc2_core_video::CVPixelBufferUnlockBaseAddress; +use objc2_foundation::NSDictionary; +use objc2_foundation::NSNumber; +use objc2_foundation::NSString; +use objc2_foundation::NSURL; +use std::path::Path; + +/// `kCVPixelFormatType_32BGRA` +const PIXEL_FORMAT_BGRA: usize = 0x4247_5241; + +/// The timescale used for the times handed to the decoder: finer than any frame rate. +const TIMESCALE: i32 = 60_000; + +/// An opened video file: its first video track, its size and its duration. +/// +/// Opening reads those facts and decodes nothing. Each [`Asset::start`] makes a new decode that +/// starts where it is told to. +pub struct Asset { + asset: Retained, + track: Retained, + width: u32, + height: u32, + duration_s: f64, +} + +impl Asset { + /// Opens a video file. + /// + /// # Errors + /// + /// Returns an error when: + /// + /// - `path` is not valid UTF-8; + /// - `path` is not a file that exists (` does not exist`); + /// - the file has no video track (` has no video track`), which is also what a file that + /// is not a video gives; + /// - `AVFoundation` does not provide the video media type. + /// + /// None of these aborts the process. + pub fn open(path: &Path) -> eyre::Result { + let text = path + .to_str() + .ok_or_else(|| eyre::Report::msg("the video path is not valid UTF-8"))?; + if !path.is_file() { + eyre::bail!("{} does not exist", path.display()); + } + + // SAFETY: plain object construction and property reads on live objects; the deprecated + // synchronous track accessor is fine for a local file + unsafe { + let url = NSURL::fileURLWithPath(&NSString::from_str(text)); + let asset = AVAsset::assetWithURL(&url); + + let media_type = AVMediaTypeVideo + .ok_or_else(|| eyre::Report::msg("AVMediaTypeVideo is unavailable"))?; + #[allow(deprecated)] + let tracks = asset.tracksWithMediaType(media_type); + let track = tracks.firstObject().ok_or_else(|| { + eyre::Report::msg(format!("{} has no video track", path.display())) + })?; + + let size = track.naturalSize(); + let duration = asset.duration(); + let duration_s = if duration.timescale > 0 && duration.value >= 0 { + duration.value as f64 / f64::from(duration.timescale) + } else { + 0.0 + }; + + Ok(Self { + asset, + track, + width: size.width.round().max(0.0) as u32, + height: size.height.round().max(0.0) as u32, + duration_s, + }) + } + } + + /// Returns the size the video was recorded at, as `(width, height)` in pixels. + #[must_use] + pub fn size(&self) -> (u32, u32) { + (self.width, self.height) + } + + /// Returns the duration `AVFoundation` reports for the file, in seconds, or `0.0` if it reports + /// none. + /// + /// For a fragmented MP4 that was never finalized (a recorder that was killed) this is the + /// playable duration, which is why recovery trusts it over `ffprobe`'s. + #[must_use] + pub fn duration_s(&self) -> f64 { + self.duration_s + } + + /// Begins decoding at `from_s` seconds into the file, up to `to_s` if given (else the end). + /// + /// The first frame returned is the one on screen at `from_s`, which can be earlier than it. A + /// negative `from_s` is `0`. The end is `to_s`, or the file's + /// [`duration_s`](Asset::duration_s) if it is `None`, and is never before `from_s`: a file + /// whose duration is unknown (`0.0`) therefore decodes little more than the frame on screen at + /// `from_s` unless `to_s` is given. The time range given to the decoder is counted in units of + /// 1/60000 of a second. A range past the end of the file is not an error: it gives at most the + /// last frame. + /// + /// Frames are converted to BGRA and scaled by the decoder to the size `scale` asks for (see + /// [`Scale`]). + /// + /// # Errors + /// + /// Returns an error when `AVFoundation` cannot make a reader for the file, rejects the output + /// configuration, or cannot start decoding. The messages carry the description `AVFoundation` + /// gives. + pub fn start(&self, from_s: f64, to_s: Option, scale: Scale) -> eyre::Result { + let from = from_s.max(0.0); + let end = to_s.unwrap_or(self.duration_s).max(from); + let (width, height) = match scale { + Scale::Native => (self.width, self.height), + Scale::Width(wide) => { + let wide = wide.clamp(2, self.width.max(2)); + let high = (u64::from(wide) * u64::from(self.height) / u64::from(self.width.max(1))) + .max(2) as u32; + // codecs like even sizes + (wide & !1, high & !1) + } + }; + + // SAFETY: every object is live; the reader is configured completely (time range, output) + // before `startReading`, which is the order AVFoundation requires + unsafe { + let reader = + AVAssetReader::assetReaderWithAsset_error(&self.asset).map_err(|error| { + eyre::Report::msg(format!( + "could not read the video: {}", + error.localizedDescription() + )) + })?; + + let start = CMTime::new((from * f64::from(TIMESCALE)).round() as i64, TIMESCALE); + let span = CMTime::new( + ((end - from) * f64::from(TIMESCALE)).ceil() as i64 + 1, + TIMESCALE, + ); + reader.setTimeRange(CMTimeRange { + start, + duration: span, + }); + + let format = NSNumber::new_usize(PIXEL_FORMAT_BGRA); + let wide = NSNumber::new_usize(width as usize); + let high = NSNumber::new_usize(height as usize); + let keys = [ + &*NSString::from_str("PixelFormatType"), + &*NSString::from_str("Width"), + &*NSString::from_str("Height"), + ]; + let values: [&AnyObject; 3] = [format.as_ref(), wide.as_ref(), high.as_ref()]; + let settings: Retained> = + NSDictionary::from_slices(&keys, &values); + + let output = AVAssetReaderTrackOutput::assetReaderTrackOutputWithTrack_outputSettings( + &self.track, + Some(&settings), + ); + // frames are only used once, in order + output.setAlwaysCopiesSampleData(false); + + let as_output: &AVAssetReaderOutput = &output; + if !reader.canAddOutput(as_output) { + eyre::bail!("the video decoder rejected the output configuration"); + } + reader.addOutput(as_output); + + if !reader.startReading() { + let why = reader.error().map_or_else( + || "unknown error".to_string(), + |error| error.localizedDescription().to_string(), + ); + eyre::bail!("could not start decoding the video: {why}"); + } + + Ok(Reader { reader, output }) + } + } +} + +/// A started decode. Dropping it cancels the decode. +/// +/// Made only by [`Asset::start`], so it has always been configured completely before reading +/// began. It moves forward only: to look at another time, start another decode. +pub struct Reader { + reader: Retained, + output: Retained, +} + +impl Reader { + /// Returns the next frame, or `None` at the end of the range. + /// + /// Frames come in order, and a sample with no picture is skipped. + /// + /// # Errors + /// + /// Returns an error when decoding fails (`decoding failed`, with `AVFoundation`'s description), + /// or the decoder returns a frame with no pixels. + pub fn next_frame(&mut self) -> eyre::Result> { + // SAFETY: the reader has been started (the only way to hold a `Reader`); the sample + // buffer and the pixel buffer it holds are live for as long as `sample` is + unsafe { + let Some(sample) = self.output.copyNextSampleBuffer() else { + return match self.reader.status() { + AVAssetReaderStatus::Failed => { + let why = self.reader.error().map_or_else( + || "unknown error".to_string(), + |error| error.localizedDescription().to_string(), + ); + Err(eyre::Report::msg(format!("decoding failed: {why}"))) + } + _ => Ok(None), + }; + }; + + let time = sample.presentation_time_stamp(); + let pts_s = if time.timescale > 0 { + time.value as f64 / f64::from(time.timescale) + } else { + 0.0 + }; + + let Some(image) = sample.image_buffer() else { + // a marker sample with no picture: skip it + return self.next_frame(); + }; + let pixels = image.as_ref(); + + CVPixelBufferLockBaseAddress(pixels, CVPixelBufferLockFlags::ReadOnly); + let width = CVPixelBufferGetWidth(pixels); + let height = CVPixelBufferGetHeight(pixels); + let stride = CVPixelBufferGetBytesPerRow(pixels); + let base = CVPixelBufferGetBaseAddress(pixels).cast::(); + + let row = width * 4; + let mut bgra = Vec::with_capacity(row * height); + if base.is_null() { + CVPixelBufferUnlockBaseAddress(pixels, CVPixelBufferLockFlags::ReadOnly); + eyre::bail!("the decoder returned a frame with no pixels"); + } + for y in 0..height { + let line = std::slice::from_raw_parts(base.add(y * stride), row); + bgra.extend_from_slice(line); + } + CVPixelBufferUnlockBaseAddress(pixels, CVPixelBufferLockFlags::ReadOnly); + + Ok(Some(Frame { + pts_s, + width: width as u32, + height: height as u32, + bgra, + })) + } + } +} + +impl Drop for Reader { + fn drop(&mut self) { + // SAFETY: cancelling a started reader is always allowed + unsafe { self.reader.cancelReading() }; + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::testkit; + + const FPS: u32 = 30; + const FRAMES: u32 = 90; + + fn video(name: &str) -> Option { + let dir = std::env::temp_dir().join(format!( + "komorebi-recorder-av-{name}-{}", + std::process::id() + )); + std::fs::create_dir_all(&dir).expect("dir"); + let path = dir.join("numbered.mp4"); + if testkit::write_numbered_video(&path, FRAMES, FPS) { + Some(path) + } else { + eprintln!("skipped: ffmpeg is not available to make a test video"); + None + } + } + + fn decode_all(asset: &Asset, from: f64, to: Option, scale: Scale) -> Vec<(f64, u32)> { + let mut reader = asset.start(from, to, scale).expect("start"); + let mut frames = Vec::new(); + while let Some(frame) = reader.next_frame().expect("frame") { + frames.push(( + frame.pts_s, + testkit::numbered_frame_index(&frame.bgra, frame.width, frame.height), + )); + } + frames + } + + #[test] + fn decoding_from_the_start_yields_every_frame_in_order() { + let Some(path) = video("all") else { return }; + let asset = Asset::open(&path).expect("open"); + + assert_eq!( + asset.size(), + (testkit::NUMBERED_WIDTH, testkit::NUMBERED_HEIGHT) + ); + assert!((asset.duration_s() - f64::from(FRAMES) / f64::from(FPS)).abs() < 0.1); + + let frames = decode_all(&asset, 0.0, None, Scale::Native); + assert_eq!(frames.len(), FRAMES as usize); + for (expected, (pts, index)) in frames.iter().enumerate() { + assert_eq!( + *index, expected as u32, + "frame {expected} was read as {index}" + ); + assert!( + (pts - expected as f64 / f64::from(FPS)).abs() < 0.002, + "frame {expected} at {pts}" + ); + } + } + + #[test] + fn starting_midway_begins_with_the_frame_on_screen_then() { + let Some(path) = video("seek") else { return }; + let asset = Asset::open(&path).expect("open"); + + // 1.0 s is frame 30; 1.02 s is still frame 30 (frame 31 appears at 1.0333) + for (from, expected) in [(1.0, 30), (1.02, 30), (2.5, 75), (0.0, 0)] { + let frames = decode_all(&asset, from, Some(from + 0.2), Scale::Native); + assert_eq!(frames[0].1, expected, "starting at {from} s"); + } + } + + #[test] + fn a_range_stops_where_it_was_asked_to() { + let Some(path) = video("range") else { return }; + let asset = Asset::open(&path).expect("open"); + + let frames = decode_all(&asset, 1.0, Some(1.5), Scale::Native); + let first = frames.first().expect("first").1; + let last = frames.last().expect("last").1; + assert_eq!(first, 30); + assert!((44..=46).contains(&last), "the range ended at frame {last}"); + } + + #[test] + fn the_decoder_scales() { + let Some(path) = video("scale") else { return }; + let asset = Asset::open(&path).expect("open"); + + let mut reader = asset + .start(1.0, Some(1.1), Scale::Width(64)) + .expect("start"); + let frame = reader.next_frame().expect("frame").expect("a frame"); + assert_eq!((frame.width, frame.height), (64, 32)); + assert_eq!(frame.bgra.len(), 64 * 32 * 4); + assert_eq!( + testkit::numbered_frame_index(&frame.bgra, frame.width, frame.height), + 30 + ); + } + + #[test] + fn reading_past_the_end_is_the_end_not_an_error() { + let Some(path) = video("past") else { return }; + let asset = Asset::open(&path).expect("open"); + + let frames = decode_all(&asset, 100.0, None, Scale::Native); + assert!(frames.len() <= 1, "{} frames past the end", frames.len()); + } + + #[test] + fn a_file_that_is_not_a_video_is_an_error_not_an_abort() { + let dir = + std::env::temp_dir().join(format!("komorebi-recorder-av-bad-{}", std::process::id())); + std::fs::create_dir_all(&dir).expect("dir"); + let text = dir.join("not-a-video.mp4"); + std::fs::write(&text, "this is not a video").expect("file"); + + assert!(Asset::open(&text).is_err()); + assert!(Asset::open(&dir.join("missing.mp4")).is_err()); + } + + /// Measures decoding a real recording: `KOMOREBI_VIDEO= cargo test ... -- --ignored + /// --nocapture`. Prints how long it takes to get the first frame at a time (the latency of + /// scrubbing) and how fast frames come once started (the rate of playing). + #[test] + #[ignore = "needs a real recording in KOMOREBI_VIDEO"] + fn real_video_decode_speed() { + use std::time::Instant; + + let Some(path) = std::env::var_os("KOMOREBI_VIDEO") else { + eprintln!("set KOMOREBI_VIDEO to a recorded video file"); + return; + }; + let opened = Instant::now(); + let asset = Asset::open(std::path::Path::new(&path)).expect("open"); + eprintln!( + "open {:?}: {}x{}, {:.1} s", + opened.elapsed(), + asset.size().0, + asset.size().1, + asset.duration_s() + ); + + for scale in [Scale::Width(320), Scale::Width(1200), Scale::Native] { + let mut first = Vec::new(); + for index in 0..8 { + let at = asset.duration_s() * (f64::from(index) + 0.5) / 8.0; + let started = Instant::now(); + let mut reader = asset.start(at, Some(at + 0.5), scale).expect("start"); + let frame = reader.next_frame().expect("frame").expect("a frame"); + first.push(started.elapsed().as_secs_f64() * 1000.0); + assert!( + frame.pts_s <= at + 0.05, + "the first frame is at or before the time asked" + ); + } + first.sort_by(f64::total_cmp); + + let started = Instant::now(); + let mut reader = asset.start(1.0, Some(3.0), scale).expect("start"); + let mut count = 0; + let mut size = (0, 0); + while let Some(frame) = reader.next_frame().expect("frame") { + count += 1; + size = (frame.width, frame.height); + } + let seconds = started.elapsed().as_secs_f64(); + eprintln!( + "{scale:?} ({}x{}): first frame median {:.0} ms (min {:.0}, max {:.0}); {count} frames in {:.2} s = {:.0} fps", + size.0, + size.1, + first[first.len() / 2], + first[0], + first[first.len() - 1], + seconds, + f64::from(count) / seconds + ); + } + } +} diff --git a/komorebi-recorder/src/av/unsupported.rs b/komorebi-recorder/src/av/unsupported.rs new file mode 100644 index 00000000..812f5eb3 --- /dev/null +++ b/komorebi-recorder/src/av/unsupported.rs @@ -0,0 +1,78 @@ +//! A host with no video decoder: opening a file says so. +//! +//! This module is built on every host that is neither macOS nor Windows, so that the investigation +//! commands (which only read a session directory) build and run there. [`Asset`] and [`Reader`] +//! have the same methods as the real ones and cannot be constructed: [`Asset::open`] always fails, +//! so nothing else here can ever be called. [`crate::av::supported`] is `false` on such a host, and +//! what needs video checks it first. Everything else in a session can still be read. + +use super::Frame; +use super::Scale; +use color_eyre::eyre; +use std::path::Path; + +fn unsupported() -> eyre::Report { + eyre::Report::msg(format!( + "decoding the recorded video is not implemented on {} yet; everything else in the \ + session can still be read", + std::env::consts::OS + )) +} + +/// An opened video file. There is no way to hold one on this host. +/// +/// The type has a field that cannot exist, so no value of it can: every method that takes `&self` +/// is unreachable. +pub struct Asset { + _never: std::convert::Infallible, +} + +impl Asset { + /// Always fails: this host has no video decoder. + /// + /// # Errors + /// + /// Always returns an error, whatever the path. The message explains that decoding the recorded + /// video is not implemented on this OS and that everything else in the session can still be + /// read. + pub fn open(_path: &Path) -> eyre::Result { + Err(unsupported()) + } + + /// Returns the size of the video. Unreachable: there is no [`Asset`] on this host. + #[must_use] + pub fn size(&self) -> (u32, u32) { + match self._never {} + } + + /// Returns the duration of the video. Unreachable: there is no [`Asset`] on this host. + #[must_use] + pub fn duration_s(&self) -> f64 { + match self._never {} + } + + /// Begins a decode. Unreachable: there is no [`Asset`] on this host. + /// + /// # Errors + /// + /// Never returns: there is no [`Asset`] to call it on. + pub fn start(&self, _from_s: f64, _to_s: Option, _scale: Scale) -> eyre::Result { + match self._never {} + } +} + +/// A started decode. There is no way to hold one on this host. +pub struct Reader { + _never: std::convert::Infallible, +} + +impl Reader { + /// Returns the next frame. Unreachable: there is no [`Reader`] on this host. + /// + /// # Errors + /// + /// Never returns: there is no [`Reader`] to call it on. + pub fn next_frame(&mut self) -> eyre::Result> { + match self._never {} + } +} diff --git a/komorebi-recorder/src/cli/mod.rs b/komorebi-recorder/src/cli/mod.rs index cfaef85b..1887b5ec 100644 --- a/komorebi-recorder/src/cli/mod.rs +++ b/komorebi-recorder/src/cli/mod.rs @@ -46,6 +46,11 @@ pub enum Command { Displays, /// Rebuild a session's timeline from its raw files, recovering it first if the recorder died Reindex(ReindexArgs), + /// Check the video-to-clock alignment against a rendered on-screen clock + Selftest(SelftestArgs), + /// The clock window used by `selftest` (internal) + #[command(hide = true)] + SelftestClock, } /// The arguments of `reindex`. @@ -55,6 +60,20 @@ pub struct ReindexArgs { pub session: PathBuf, } +/// The arguments of `selftest`. +#[derive(Args, Debug)] +pub struct SelftestArgs { + /// Seconds to record + #[arg(long, default_value_t = 5.0)] + pub seconds: f64, + /// Largest allowed disagreement between the video and the clocks, in milliseconds + #[arg(long, default_value_t = 100.0)] + pub tolerance_ms: f64, + /// Session directory for the self-test recording + #[arg(long)] + pub out: Option, +} + impl Command { /// Whether this command was asked for JSON, which also decides how a failure is reported. #[must_use] @@ -98,6 +117,8 @@ pub fn run(command: Command) -> eyre::Result<()> { Command::Record(args) => record::record(&args), Command::Displays => record::displays(), Command::Reindex(args) => reindex(&args), + Command::Selftest(args) => crate::selftest::run(&args), + Command::SelftestClock => crate::selftest::run_clock(), } } diff --git a/komorebi-recorder/src/lib.rs b/komorebi-recorder/src/lib.rs index 2849d765..6e16b5c1 100644 --- a/komorebi-recorder/src/lib.rs +++ b/komorebi-recorder/src/lib.rs @@ -56,6 +56,7 @@ //! | [`recovery`] | rebuilding a session whose recorder died | //! | [`cli`] | the command line | //! | [`app`] | the recorder window | +//! | [`av`] and [`selftest`] | decoding the recorded video and checking it against the clocks | //! //! # What is not supported //! @@ -71,6 +72,7 @@ pub mod analysis; pub mod annotations; pub mod app; +pub mod av; pub mod capture; pub mod cli; pub mod clock; @@ -82,6 +84,7 @@ pub mod platform; pub mod power; pub mod recorder; pub mod recovery; +pub mod selftest; pub mod session; pub mod sources; pub mod system_state; diff --git a/komorebi-recorder/src/platform/win/capture.rs b/komorebi-recorder/src/platform/win/capture.rs index 099563dc..be50c547 100644 --- a/komorebi-recorder/src/platform/win/capture.rs +++ b/komorebi-recorder/src/platform/win/capture.rs @@ -214,12 +214,12 @@ pub fn main_display() -> eyre::Result { /// timer and can lag behind (or run ahead of) what actually reached the video. A file whose /// duration it cannot read (some truncated ones), or that is not longer than zero, is `None`, and /// recovery says it could not tell. On macOS the duration comes from `AVFoundation`. -/// -/// Media Foundation's reader is not part of the build yet, so this is always `None`. #[must_use] pub fn media_duration_s(path: &Path) -> Option { - let _ = path; - None + crate::av::Asset::open(path) + .ok() + .map(|asset| asset.duration_s()) + .filter(|seconds| *seconds > 0.0) } /// The [`CaptureBackend`](crate::capture::CaptureBackend) of this host: Windows.Graphics.Capture. diff --git a/komorebi-recorder/src/selftest.rs b/komorebi-recorder/src/selftest.rs new file mode 100644 index 00000000..23b7166c --- /dev/null +++ b/komorebi-recorder/src/selftest.rs @@ -0,0 +1,844 @@ +//! The sync self-test: prove that "video second N" really is "clock time N". +//! +//! A child process (so the recorder does not exclude it from the capture along with its own +//! windows) draws a large live clock plus a strip of 40 black/white cells that binary-encodes the +//! host clock in milliseconds at the moment each frame was painted. The parent records the screen +//! headlessly, then for several video timestamps decodes the strip out of the extracted frame and +//! compares it with the time the session's manifest predicts for that video second. Nothing is +//! read by eye or OCR. +//! +//! What the comparison includes beyond the recorder's own accuracy: the clock is painted a +//! little before the compositor shows it and SCK frames are rate-limited, so the visible clock +//! trails the video time by one to a few frame intervals. That is why the default tolerance is +//! 100 ms and the mean bias is reported separately. +//! +//! # The pieces +//! +//! | Item | Is | +//! |---|---| +//! | [`run`] | the `selftest` command: records, decodes, compares, and stores the verdict in the manifest | +//! | [`run_clock`] | the child process: the window that paints the clock and the strip (the hidden `selftest-clock` command) | +//! | [`encode_cells`], [`decode_cells`] | the strip: a clock value to cell colors, and the average luminance of each cell back to a clock value | +//! | [`strip_filter`], [`strip_units`] | where the strip is in a display's video, as an `ffmpeg` crop | +//! | [`wrapped_difference_ms`], [`evaluate`] | the comparison, and the verdict ([`Reading`] in, [`Calibration`] out) | +//! +//! The pure parts (the strip, the geometry, the comparison) are the same on both hosts and are +//! tested without a screen. [`run`] and [`run_clock`] need a screen, a window system and `ffmpeg`, +//! so no automated test covers them: the `selftest` command is the check, and +//! `docs/dev/windows-checkpoints.md` (W2) says what to expect of it on Windows. +//! +//! # The strip +//! +//! The strip is 40 cells, one for each bit of the master clock in milliseconds, most significant +//! first, white for 1 and black for 0, of the value taken modulo 2^40 (a little under 35 years of +//! milliseconds, so it does not wrap in practice). It is painted at the same place in the clock +//! window every frame, and [`strip_filter`] says where that is in a display's video. The window +//! also shows the master clock and the wall clock as text, for a person looking at the video. +//! +//! # Hosts +//! +//! The recording itself is the host's capture. The geometry differs: on macOS the window's +//! position and the display's frame are both in points, and the video is a number of pixels to the +//! point; on Windows the display's frame is physical pixels and the window's position is still in +//! points. [`strip_units`] brings them to common units. +//! +//! # Example +//! +//! The strip carries a clock reading, and the difference between what the video shows and what the +//! manifest predicts is the error: +//! +//! ```rust +//! use komorebi_recorder::selftest::{decode_cells, encode_cells, evaluate, Reading}; +//! +//! // the clock at the moment a frame was painted, in ms: encoded as cells, then read back from +//! // the average luminance of each cell in the video +//! let painted_ms = 10_000; +//! let luminance: Vec = encode_cells(painted_ms) +//! .iter() +//! .map(|white| if *white { 255 } else { 0 }) +//! .collect(); +//! let observed_ms = decode_cells(&luminance).expect("a legible strip"); +//! assert_eq!(observed_ms, painted_ms); +//! +//! // the manifest predicted the video second to be at 10 030 ms: the video is 30 ms behind +//! let verdict = evaluate( +//! &[Reading { video_s: 1.0, predicted_ms: 10_030, observed_ms }], +//! 100.0, +//! ); +//! assert!(verdict.passed); +//! assert_eq!(verdict.samples[0].error_ms, -30.0); +//! ``` + +use crate::cli::SelftestArgs; +use crate::clock; +use crate::frames; +use crate::platform::capture; +use crate::recorder; +use crate::recorder::Live; +use crate::recorder::OsLogOptions; +use crate::recorder::RecordOptions; +use crate::session::Calibration; +use crate::session::CalibrationSample; +use crate::session::Manifest; +use crate::session::Session; +use crate::timecode; +use color_eyre::eyre; +use color_eyre::eyre::WrapErr; +use eframe::egui; +use std::io::BufRead; +use std::io::BufReader; +use std::io::Write; +use std::process::Command; +use std::process::Stdio; +use std::time::Duration; +use std::time::Instant; + +/// Number of cells in the strip, i.e. bits of the encoded millisecond clock. +const CELLS: usize = 40; +const CELL_WIDTH: f32 = 24.0; +const CELL_HEIGHT: f32 = 60.0; +/// Where the strip sits inside the clock window, in points. +const STRIP_X: f32 = 20.0; +const STRIP_Y: f32 = 110.0; +const WINDOW_SIZE: [f32; 2] = [1000.0, 260.0]; +const WINDOW_POSITION: [f32; 2] = [80.0, 80.0]; + +const WRAP_MS: u64 = 1 << CELLS; + +/// Returns the cell colors (true = white) for a clock value in milliseconds, most significant +/// first. +/// +/// There are 40 cells, so the value is taken modulo 2^40: bits above the lowest 40 are not shown. +/// +/// # Example +/// +/// ```rust +/// use komorebi_recorder::selftest::encode_cells; +/// +/// // 5 is 101 in binary: the last three cells, with the rest black +/// let cells = encode_cells(5); +/// assert_eq!(cells.len(), 40); +/// assert!(cells[..37].iter().all(|white| !white)); +/// assert_eq!(cells[37..], [true, false, true]); +/// ``` +#[must_use] +pub fn encode_cells(ms: u64) -> [bool; CELLS] { + let mut cells = [false; CELLS]; + for (index, cell) in cells.iter_mut().enumerate() { + *cell = (ms >> (CELLS - 1 - index)) & 1 == 1; + } + cells +} + +/// Recovers the clock value from the average luminance of each cell, 0 (black) to 255 (white). +/// +/// A cell at 128 or above is white. Returns `None` if there are not exactly 40 values, or if more +/// than 3 of them are neither near black nor near white (from 64 up to, but not including, 192), +/// which is what the average of a crop that missed the strip looks like. Cells that are soft but +/// still clearly one or the other, as anti-aliasing and video compression make them, decode. +/// +/// # Example +/// +/// ```rust +/// use komorebi_recorder::selftest::{decode_cells, encode_cells}; +/// +/// // a clock value survives encoding and reading back from the cells' luminance +/// let ms = 1_210_934_273; +/// let sharp: Vec = encode_cells(ms).iter().map(|white| if *white { 255 } else { 0 }).collect(); +/// assert_eq!(decode_cells(&sharp), Some(ms)); +/// +/// // soft edges still decode +/// let soft: Vec = sharp.iter().map(|level| if *level == 255 { 230 } else { 12 }).collect(); +/// assert_eq!(decode_cells(&soft), Some(ms)); +/// +/// // flat gray is a crop that missed the strip, and a short crop is not a strip +/// assert_eq!(decode_cells(&[128; 40]), None); +/// assert_eq!(decode_cells(&[0; 10]), None); +/// ``` +#[must_use] +pub fn decode_cells(luminance: &[u8]) -> Option { + if luminance.len() != CELLS { + return None; + } + + // a legible strip is overwhelmingly near-black or near-white; a mid-grey average means the + // crop missed the strip + let ambiguous = luminance + .iter() + .filter(|value| (64..192).contains(*value)) + .count(); + if ambiguous > 3 { + return None; + } + + Some(luminance.iter().fold(0_u64, |value, level| { + (value << 1) | u64::from(*level >= 128) + })) +} + +/// Returns `observed - predicted` in milliseconds, accounting for the strip's wrap-around. +/// +/// The strip holds a clock value modulo 2^40 milliseconds, so two values on either side of a wrap +/// are close, however far apart they are as numbers. The result is the shorter way round: within +/// half the wrap, in either direction. +/// +/// # Example +/// +/// ```rust +/// use komorebi_recorder::selftest::wrapped_difference_ms; +/// +/// assert_eq!(wrapped_difference_ms(1_050, 1_000), 50); +/// assert_eq!(wrapped_difference_ms(950, 1_000), -50); +/// +/// // the strip wrapped between the prediction and the observation +/// let wrap = 1_u64 << 40; +/// assert_eq!(wrapped_difference_ms(5, wrap - 5), 10); +/// assert_eq!(wrapped_difference_ms(wrap - 5, 5), -10); +/// ``` +#[must_use] +pub fn wrapped_difference_ms(observed: u64, predicted: u64) -> i64 { + let wrapped = (observed.wrapping_sub(predicted)) % WRAP_MS; + if wrapped >= WRAP_MS / 2 { + wrapped as i64 - WRAP_MS as i64 + } else { + wrapped as i64 + } +} + +/// Returns the ffmpeg `-vf` expression that crops the strip out of a display's video and averages +/// each cell down to one gray pixel. +/// +/// `window_origin` is the clock window's top left corner and `display_origin` the display's, in +/// the same units, and `scale` is how many pixels of video one of those units is. The crop is the +/// strip's position in the window (20, 110) and its size (40 cells of 24 by 60) in those units, +/// multiplied by `scale`, and cut down to the middle half of the cells' height so that the edges +/// of the strip do not blur into the average. The cells are then scaled to 40 by 1 pixels with area +/// averaging. +/// +/// # Example +/// +/// ```rust +/// use komorebi_recorder::selftest::strip_filter; +/// +/// // the window at (80, 80) on a display at the origin, one pixel to the unit +/// assert_eq!( +/// strip_filter((80.0, 80.0), (0.0, 0.0), 1.0), +/// "crop=960:30:100:205,scale=40:1:flags=area" +/// ); +/// +/// // the same window on a display to the left, two pixels to the unit +/// assert_eq!( +/// strip_filter((-1720.0, 80.0), (-1800.0, 0.0), 2.0), +/// "crop=1920:60:200:410,scale=40:1:flags=area" +/// ); +/// ``` +#[must_use] +pub fn strip_filter(window_origin: (f64, f64), display_origin: (f64, f64), scale: f64) -> String { + let x = ((window_origin.0 - display_origin.0) + f64::from(STRIP_X)) * scale; + let y = ((window_origin.1 - display_origin.1) + f64::from(STRIP_Y)) * scale; + let width = f64::from(CELL_WIDTH) * CELLS as f64 * scale; + // sample the middle of the cells vertically + let height = f64::from(CELL_HEIGHT) * 0.5 * scale; + let y = y + f64::from(CELL_HEIGHT) * 0.25 * scale; + + format!( + "crop={}:{}:{}:{},scale={CELLS}:1:flags=area", + width.round() as i64, + height.round().max(1.0) as i64, + x.round() as i64, + y.round() as i64, + ) +} + +/// Where the strip is in a display's video, as `strip_filter` wants it: the window's and the +/// display's origins in the same units, and how many pixels of video one of those units is. +/// +/// On a Mac the window (egui's points) and the display (CoreGraphics' points) share units and the +/// video is `video_scale` pixels to the point. On Windows the video is the display's physical +/// pixels and the window is still in points, `pixels_per_point` of which make a pixel, so the +/// display's origin is brought into points and the unit is the pixels-per-point. +/// +/// `display_frame_is_pixels` says which: true where the display's frame is in physical pixels +/// (Windows). A `pixels_per_point` below 0.1 is taken as 0.1. +/// +/// # Example +/// +/// ```rust +/// use komorebi_recorder::selftest::{strip_filter, strip_units}; +/// +/// // a Mac: both in points, the video 2 pixels to the point +/// assert_eq!(strip_units((100.0, 50.0), 2.0, 2.0, false), ((100.0, 50.0), 2.0)); +/// +/// // Windows at 150 %: the display at 1920 px is at 1280 points, and a point is 1.5 pixels +/// let ((x, y), unit) = strip_units((1920.0, 0.0), 1.0, 1.5, true); +/// assert_eq!((x, y, unit), (1280.0, 0.0, 1.5)); +/// +/// // so a window 80 points into that display puts its strip 150 pixels in +/// let filter = strip_filter((1280.0 + 80.0, 80.0), (x, y), unit); +/// assert!(filter.starts_with("crop=1440:45:150:")); +/// ``` +#[must_use] +pub fn strip_units( + display_frame_origin: (f64, f64), + video_scale: f64, + pixels_per_point: f64, + display_frame_is_pixels: bool, +) -> ((f64, f64), f64) { + if display_frame_is_pixels { + let ppp = pixels_per_point.max(0.1); + ( + (display_frame_origin.0 / ppp, display_frame_origin.1 / ppp), + ppp, + ) + } else { + (display_frame_origin, video_scale) + } +} + +/// The child process: a borderless, untitled (so komorebi leaves it alone) window that paints +/// the clock as fast as it can. +/// +/// The window is 1000 by 260 points, always on top, at (80, 80). It paints the master clock and the +/// wall clock as text, and the strip (see the [module docs](self#the-strip)) from a clock reading +/// taken as each frame's content is decided. It prints a line `RECT +/// ` on its standard output whenever its inner rectangle changes, which is how +/// [`run`] finds it. +/// +/// This blocks until the window is closed, or the parent kills it. It exits the process after 120 +/// seconds, in case the parent died without killing it. +/// +/// # Errors +/// +/// Returns an error if the window system cannot create the window (`the clock window failed`). +/// +/// This opens a window, so it has no example and no automated test; it runs when [`run`] starts it +/// as the hidden `selftest-clock` command. +pub fn run_clock() -> eyre::Result<()> { + let options = eframe::NativeOptions { + viewport: egui::ViewportBuilder::default() + .with_title("") + .with_decorations(false) + .with_position(WINDOW_POSITION) + .with_inner_size(WINDOW_SIZE) + .with_always_on_top() + .with_active(false), + ..Default::default() + }; + + eframe::run_native( + "komorebi-recorder-clock", + options, + Box::new(|_| Ok(Box::new(ClockApp::default()))), + ) + .map_err(|error| eyre::Report::msg(format!("the clock window failed: {error}"))) +} + +#[derive(Default)] +struct ClockApp { + last_rect: Option, + started: Option, +} + +impl eframe::App for ClockApp { + fn update(&mut self, context: &egui::Context, _frame: &mut eframe::Frame) { + let started = *self.started.get_or_insert_with(Instant::now); + // never linger if the parent dies without killing us + if started.elapsed() > Duration::from_secs(120) { + std::process::exit(0); + } + + // read the clock first: this is the moment the frame's content is decided + let mach_ns = clock::mono_now_ns(); + let wall_us = clock::wall_now_us(); + let ms = (mach_ns / 1_000_000) % WRAP_MS; + + if let Some(rect) = context.input(|input| input.viewport().inner_rect) + && self.last_rect != Some(rect) + { + self.last_rect = Some(rect); + println!( + "RECT {} {} {} {} {}", + rect.min.x, + rect.min.y, + rect.width(), + rect.height(), + context.pixels_per_point() + ); + let _ = std::io::stdout().flush(); + } + + egui::CentralPanel::default() + .frame(egui::Frame::NONE.fill(egui::Color32::from_gray(40))) + .show(context, |ui| { + let painter = ui.painter(); + painter.text( + egui::pos2(20.0, 14.0), + egui::Align2::LEFT_TOP, + format!("mach {mach_ns} ns"), + egui::FontId::monospace(30.0), + egui::Color32::WHITE, + ); + painter.text( + egui::pos2(20.0, 56.0), + egui::Align2::LEFT_TOP, + crate::timeline::format_wall_us(wall_us).unwrap_or_default(), + egui::FontId::monospace(30.0), + egui::Color32::LIGHT_GREEN, + ); + + for (index, white) in encode_cells(ms).into_iter().enumerate() { + let cell = egui::Rect::from_min_size( + egui::pos2(STRIP_X + index as f32 * CELL_WIDTH, STRIP_Y), + egui::vec2(CELL_WIDTH, CELL_HEIGHT), + ); + painter.rect_filled( + cell, + 0.0, + if white { + egui::Color32::WHITE + } else { + egui::Color32::BLACK + }, + ); + } + }); + + context.request_repaint(); + } +} + +/// One decoded sample, before it is turned into a [`CalibrationSample`]. +#[derive(Debug, Clone, PartialEq)] +pub struct Reading { + /// The video time the frame was taken at, in seconds. + pub video_s: f64, + /// The clock value the manifest predicts for that video time, in milliseconds on the master + /// clock modulo 2^40. + pub predicted_ms: u64, + /// The clock value read from the strip in the frame, in the same units. + pub observed_ms: u64, +} + +/// Turns readings into a calibration verdict. +/// +/// Each reading's error is the observed clock minus the predicted one, in milliseconds +/// ([`wrapped_difference_ms`]), so a negative error means the video shows a clock that is behind +/// what the manifest predicts. The verdict passes if there is at least one reading and the largest +/// error, in either direction, is no more than `tolerance_ms`. The verdict stores the readings as +/// samples, with the times in nanoseconds on the master clock. +/// +/// # Example +/// +/// ```rust +/// use komorebi_recorder::selftest::{evaluate, Reading}; +/// +/// let readings = vec![ +/// Reading { video_s: 1.0, predicted_ms: 10_000, observed_ms: 9_970 }, +/// Reading { video_s: 1.5, predicted_ms: 10_500, observed_ms: 10_460 }, +/// ]; +/// +/// let verdict = evaluate(&readings, 100.0); +/// assert!(verdict.passed); +/// assert_eq!(verdict.max_abs_error_ms, 40.0); +/// assert_eq!(verdict.samples[0].error_ms, -30.0); +/// +/// // the same readings against a tighter tolerance +/// assert!(!evaluate(&readings, 35.0).passed); +/// +/// // no readings is not a pass +/// assert!(!evaluate(&[], 100.0).passed); +/// ``` +#[must_use] +pub fn evaluate(readings: &[Reading], tolerance_ms: f64) -> Calibration { + let samples: Vec = readings + .iter() + .map(|reading| { + let error_ms = wrapped_difference_ms(reading.observed_ms, reading.predicted_ms) as f64; + CalibrationSample { + video_s: reading.video_s, + predicted_mach_ns: reading.predicted_ms * 1_000_000, + observed_mach_ns: reading.observed_ms * 1_000_000, + error_ms, + } + }) + .collect(); + + let max_abs_error_ms = samples + .iter() + .map(|sample| sample.error_ms.abs()) + .fold(0.0, f64::max); + + Calibration { + method: "on-screen binary clock strip decoded from extracted video frames".to_string(), + passed: !samples.is_empty() && max_abs_error_ms <= tolerance_ms, + max_abs_error_ms, + tolerance_ms, + samples, + } +} + +/// Decode the strip from one frame of one display file. +fn read_strip( + session: &Session, + file: &str, + file_s: f64, + filter: &str, +) -> eyre::Result> { + let output = Command::new("ffmpeg") + .args(["-v", "error", "-ss", &format!("{file_s:.3}"), "-i"]) + .arg(session.path(file)) + .args(["-frames:v", "1", "-vf", filter]) + .args(["-pix_fmt", "gray", "-f", "rawvideo", "-"]) + .output() + .wrap_err("could not run ffmpeg")?; + + if !output.status.success() { + eyre::bail!( + "ffmpeg failed at {}: {}", + timecode::format(file_s), + String::from_utf8_lossy(&output.stderr).trim() + ); + } + + Ok(decode_cells(&output.stdout)) +} + +/// `[x, y, width, height, pixels_per_point]`, the window in points. +type WindowReport = [f64; 5]; + +fn spawn_clock() -> eyre::Result<( + std::process::Child, + crossbeam_channel::Receiver, +)> { + let mut child = Command::new(std::env::current_exe()?) + .arg("selftest-clock") + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .spawn() + .wrap_err("could not start the clock window")?; + + let stdout = child + .stdout + .take() + .ok_or_else(|| eyre::Report::msg("the clock window has no stdout"))?; + + let (tx, rx) = crossbeam_channel::unbounded(); + std::thread::spawn(move || { + for line in BufReader::new(stdout).lines().map_while(Result::ok) { + let numbers: Vec = line + .strip_prefix("RECT ") + .map(|rest| { + rest.split_whitespace() + .filter_map(|n| n.parse().ok()) + .collect() + }) + .unwrap_or_default(); + + match numbers[..] { + [x, y, w, h, ppp] => { + let _ = tx.send([x, y, w, h, ppp]); + } + // an older clock window did not say + [x, y, w, h] => { + let _ = tx.send([x, y, w, h, 1.0]); + } + _ => {} + } + } + }); + + Ok((child, rx)) +} + +/// Runs the self-test: records the screen, reads the clock strip out of the video, and compares +/// it with what the session's manifest predicts. +/// +/// It starts the clock window as a child process ([`run_clock`], through the hidden +/// `selftest-clock` command of this same executable) on the main display, waits for it to say where +/// it is, and records that display alone for `args.seconds` seconds with no komorebi and no OS +/// trace. Then, for every half second of video from 1 s to half a second before the end, it asks +/// `ffmpeg` for that frame, reads the clock out of the strip, and compares it with the clock the +/// manifest predicts for that video second. A moment at which the strip cannot be read is not a +/// reading, and is noted in the manifest's warnings. The verdict ([`evaluate`]) is stored in the +/// manifest as `calibration`, a table of the errors is printed, and the session directory is +/// printed last. +/// +/// The recording is kept: `args.out` is where it goes, and by default it is next to the other +/// recordings. What is printed is more than a pass or a fail: the error of each sample, and their +/// mean, the bias, which is reported apart from the worst error because the clock is painted +/// slightly before it is shown. +/// +/// # Errors +/// +/// Returns an error if +/// +/// * `ffmpeg` is not on `PATH`; +/// * the OS has not let this process capture the screen (on macOS, the Screen Recording +/// permission), or the main display cannot be found; +/// * the clock window cannot be started, or does not report its position within 20 seconds; +/// * the recording fails, for any reason `record` would fail; +/// * no video was recorded for the main display; +/// * the clock cannot be found in any frame (is the clock window covered, or moved?): a frame is +/// saved under the session's `frames/` directory for a look; +/// * the manifest cannot be written; +/// * the test **fails**: the video disagrees with the clocks by more than `args.tolerance_ms`. The +/// verdict is in the manifest by then. +/// +/// # Example +/// +/// ```rust,no_run +/// use komorebi_recorder::cli::SelftestArgs; +/// use komorebi_recorder::selftest::run; +/// +/// // record for 8 seconds, and allow the video to disagree with the clocks by up to 100 ms +/// run(&SelftestArgs { seconds: 8.0, tolerance_ms: 100.0, out: None })?; +/// # Ok::<(), color_eyre::eyre::Report>(()) +/// ``` +/// +/// This needs a screen, a window system and `ffmpeg`, so it is not run, and no automated test +/// covers it: `komorebi-recorder selftest` is the check. +pub fn run(args: &SelftestArgs) -> eyre::Result<()> { + if !frames::ffmpeg_available() { + eyre::bail!("the self-test decodes frames with ffmpeg, which was not found on PATH"); + } + + capture::ensure_permission()?; + + let display = capture::main_display()?; + let main_id = display.id; + + eprintln!("starting the clock window on display {main_id}..."); + let (mut child, rect_rx) = spawn_clock()?; + + let first = rect_rx + .recv_timeout(Duration::from_secs(20)) + .map_err(|_| eyre::Report::msg("the clock window never reported its position"))?; + + // let the window settle (and be shown) before trusting its position + std::thread::sleep(Duration::from_millis(1500)); + let mut rect = first; + while let Ok(latest) = rect_rx.try_recv() { + rect = latest; + } + + let options = RecordOptions { + out: args.out.clone(), + displays: vec![main_id], + oslog: OsLogOptions { + enabled: false, + ..OsLogOptions::default() + }, + duration: Some(Duration::from_secs_f64(args.seconds)), + no_komorebi: true, + ..RecordOptions::default() + }; + + eprintln!("recording {:.0}s...", args.seconds); + let live = Live::new(); + let recorded = recorder::record(&options, &live); + let _ = child.kill(); + let _ = child.wait(); + let dir = recorded?; + + let session = Session::open(&dir)?; + let mut manifest = session.manifest()?; + let entry = manifest + .displays + .iter() + .find(|entry| entry.result.display.id == main_id && entry.result.frames > 0) + .cloned() + .ok_or_else(|| eyre::Report::msg("no video was recorded for the main display"))?; + + let (display_origin, video_per_unit) = strip_units( + (display.frame[0], display.frame[1]), + entry.result.scale, + rect[4], + cfg!(windows), + ); + let filter = strip_filter((rect[0], rect[1]), display_origin, video_per_unit); + + let (_, duration) = manifest.video_span(); + let mut readings = Vec::new(); + let mut undecodable = Vec::new(); + let mut video_s = 1.0; + while video_s <= duration - 0.5 { + let file_s = video_s - entry.offset_s; + let predicted_ms = manifest.mach_at_video_seconds(video_s) / 1_000_000 % WRAP_MS; + match read_strip(&session, &entry.result.file, file_s, &filter)? { + Some(observed_ms) => readings.push(Reading { + video_s, + predicted_ms, + observed_ms, + }), + None => undecodable.push(video_s), + } + video_s += 0.5; + } + + if readings.is_empty() { + // leave a frame behind so the problem can be looked at + let _ = frames::extract(&session, &manifest, &[1.0], Some(main_id)); + eyre::bail!( + "could not find the clock in the video (is the clock window covered, or was it moved? \ + window reported at {rect:?}); a frame was saved under {}", + session.path("frames").display() + ); + } + + let calibration = evaluate(&readings, args.tolerance_ms); + report(&calibration, &undecodable, &mut manifest); + manifest.calibration = Some(calibration.clone()); + session.write_manifest(&manifest)?; + println!("session: {}", dir.display()); + + if calibration.passed { + Ok(()) + } else { + Err(eyre::Report::msg(format!( + "sync self-test FAILED: the video disagrees with the clocks by up to {:.0} ms (tolerance {:.0} ms)", + calibration.max_abs_error_ms, calibration.tolerance_ms + ))) + } +} + +fn report(calibration: &Calibration, undecodable: &[f64], manifest: &mut Manifest) { + println!("video second | clock shown in video vs predicted"); + for sample in &calibration.samples { + println!( + " {:>6} | {:+.0} ms", + timecode::format(sample.video_s), + sample.error_ms + ); + } + + let mean = calibration + .samples + .iter() + .map(|sample| sample.error_ms) + .sum::() + / calibration.samples.len().max(1) as f64; + println!( + "{} samples, mean bias {mean:+.1} ms (the clock is painted slightly before it is shown), \ + worst {:.0} ms, tolerance {:.0} ms: {}", + calibration.samples.len(), + calibration.max_abs_error_ms, + calibration.tolerance_ms, + if calibration.passed { "PASS" } else { "FAIL" } + ); + + if !undecodable.is_empty() { + let note = format!( + "self-test could not read the clock at {} of {} sampled timestamps", + undecodable.len(), + undecodable.len() + calibration.samples.len() + ); + println!("note: {note}"); + manifest.warnings.push(note); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn luminance(ms: u64) -> Vec { + encode_cells(ms) + .iter() + .map(|white| if *white { 255 } else { 0 }) + .collect() + } + + #[test] + fn a_mac_window_and_display_share_units_and_a_windows_window_does_not() { + // a Mac: both in points, the video 2 pixels to the point + assert_eq!( + strip_units((100.0, 50.0), 2.0, 2.0, false), + ((100.0, 50.0), 2.0) + ); + + // Windows at 150 %: the display at 1920 px is 1280 points, and a point is 1.5 pixels + let ((x, y), unit) = strip_units((1920.0, 0.0), 1.0, 1.5, true); + assert_eq!((x, y, unit), (1280.0, 0.0, 1.5)); + + // so a window 80 points into that display puts its strip 120 pixels + 20 points in + let filter = strip_filter((1280.0 + 80.0, 80.0), (x, y), unit); + assert!(filter.starts_with("crop=1440:45:150:"), "{filter}"); + } + + #[test] + fn the_strip_roundtrips_any_clock_value() { + for ms in [ + 0, + 1, + 0xA5A5A5A5A5 & (WRAP_MS - 1), + 1_210_934_273, + WRAP_MS - 1, + ] { + assert_eq!(decode_cells(&luminance(ms)), Some(ms), "{ms}"); + } + } + + #[test] + fn decoding_tolerates_antialiasing_but_not_a_missed_crop() { + // slightly soft edges still decode + let mut soft = luminance(0b1011_0010); + for (index, level) in soft.iter_mut().enumerate() { + *level = if *level == 255 { + 230 + (index % 5) as u8 + } else { + 10 + (index % 7) as u8 + }; + } + assert_eq!(decode_cells(&soft), Some(0b1011_0010)); + + // flat grey is a crop that missed the strip + assert_eq!(decode_cells(&[128; CELLS]), None); + assert_eq!(decode_cells(&[0; 10]), None, "wrong length"); + } + + #[test] + fn wrap_around_differences_are_small() { + assert_eq!(wrapped_difference_ms(1_050, 1_000), 50); + assert_eq!(wrapped_difference_ms(950, 1_000), -50); + // the strip wrapped between the prediction and the observation + assert_eq!(wrapped_difference_ms(5, WRAP_MS - 5), 10); + assert_eq!(wrapped_difference_ms(WRAP_MS - 5, 5), -10); + } + + #[test] + fn the_crop_filter_targets_the_strip_in_display_pixels() { + // window at (80, 80) on a display at the origin, 1x + assert_eq!( + strip_filter((80.0, 80.0), (0.0, 0.0), 1.0), + "crop=960:30:100:205,scale=40:1:flags=area" + ); + // the same window on a display to the left, at 2x + assert_eq!( + strip_filter((-1720.0, 80.0), (-1800.0, 0.0), 2.0), + "crop=1920:60:200:410,scale=40:1:flags=area" + ); + } + + #[test] + fn evaluation_passes_within_tolerance_and_reports_the_worst_error() { + let readings = vec![ + Reading { + video_s: 1.0, + predicted_ms: 10_000, + observed_ms: 9_970, + }, + Reading { + video_s: 1.5, + predicted_ms: 10_500, + observed_ms: 10_460, + }, + ]; + let calibration = evaluate(&readings, 100.0); + assert!(calibration.passed); + assert!((calibration.max_abs_error_ms - 40.0).abs() < 1e-9); + assert_eq!(calibration.samples[0].error_ms, -30.0); + + assert!(!evaluate(&readings, 35.0).passed); + assert!(!evaluate(&[], 100.0).passed, "no readings is not a pass"); + } +}