diff --git a/Cargo.lock b/Cargo.lock index a07ef11a..8476cc76 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1914,6 +1914,7 @@ dependencies = [ "objc2-core-foundation", "objc2-core-graphics", "objc2-core-media", + "objc2-core-video", "objc2-foundation 0.3.2", "objc2-screen-capture-kit", "parking_lot", diff --git a/docs/PLATFORM.md b/docs/PLATFORM.md index a21c8ce0..cb6a7331 100644 --- a/docs/PLATFORM.md +++ b/docs/PLATFORM.md @@ -102,6 +102,12 @@ the locked file. The wire protocol is the same on both. The recorder's own binary does not depend on either komorebi's crates. +### Decoding the recorded video + +macOS decodes with AVFoundation's `AVAssetReader`, Windows with Media Foundation's Source Reader, and the `unsupported` +host with nothing: it says so when asked to open a file. Both real hosts are asked for BGRA frames at a chosen size, so +that a contact sheet never decodes more pixels than it shows. + ### The program's own windows The recorder window and the viewer must not be in the video they are recording, and komorebi must not tile them. diff --git a/docs/usage/recording-a-bug-repro.md b/docs/usage/recording-a-bug-repro.md index f64566ba..199e1467 100644 --- a/docs/usage/recording-a-bug-repro.md +++ b/docs/usage/recording-a-bug-repro.md @@ -54,6 +54,8 @@ The recorder window keeps itself out of the video (Windows 10 version 2004 or la - **komorebi's log level.** The recorder never changes komorebi's log level or lifecycle; it only warns. For a useful repro, start komorebi with `komorebi --log-level trace` (or `RUST_LOG=komorebi=trace`) first. At the default `info` level most of what the window manager _decided_ is not in the log. +- **`ffmpeg`** is only needed for `frames` and `selftest`. It is never used to capture. The `nix develop` shell provides + it; otherwise `brew install ffmpeg`. The recorder window is borderless and untitled on purpose: komorebi does not manage windows without a title, so opening it does not move your layout around while you are trying to reproduce a layout bug. It is also excluded from the video, as is any other `komorebi-recorder` process. @@ -167,6 +169,9 @@ anchors sampled every 5 seconds during the recording and at every sleep, so cloc the event, so the stamp is when the handler **finished**; the `processing event` line in `komorebi.log` is when it began. +`komorebi-recorder selftest` checks this on your machine: it records a rendered on-screen clock and compares the clock +visible in the video with the time the session predicts. The result is stored in `session.json` under `calibration`. + ## Things to know - **What can still be missing.** Events from a `log stream` during the moment it was dead and being restarted; komorebi diff --git a/komorebi-recorder/Cargo.toml b/komorebi-recorder/Cargo.toml index e5db3af8..4aae5819 100644 --- a/komorebi-recorder/Cargo.toml +++ b/komorebi-recorder/Cargo.toml @@ -68,6 +68,14 @@ objc2-core-media = { version = "0.3", default-features = false, features = [ "CMTime", "CMTimeRange", ] } +objc2-core-video = { version = "0.3", default-features = false, features = [ + "std", + "CVBase", + "CVBuffer", + "CVImageBuffer", + "CVPixelBuffer", + "CVReturn", +] } objc2-foundation = { version = "0.3", default-features = false, features = [ "std", "NSArray", diff --git a/komorebi-recorder/src/av/mf.rs b/komorebi-recorder/src/av/mf.rs new file mode 100644 index 00000000..85fc1c3a --- /dev/null +++ b/komorebi-recorder/src/av/mf.rs @@ -0,0 +1,368 @@ +//! Decoding recorded video on Windows with Media Foundation's Source Reader. +//! +//! The reader is asked for BGRA frames at the size wanted (its video processor converts and +//! scales, so a contact sheet never decodes more pixels than it shows), seeks to the key frame +//! before the time asked for, and decodes forward; [`super::Selector`] says which of the decoded +//! frames are the answer. Media Foundation objects belong to the thread that made them as far as +//! this code is concerned, so a decode is started and read on one thread, as the Mac's are. +//! +//! An [`Asset`] keeps the path and what was read from the file when it was opened (size, duration, +//! whether it is HEVC), not an open reader: every [`Asset::start`] opens a new Source Reader, so a +//! file that changes after it was opened is read as it is then. +//! +//! The recorder writes H.264 on Windows, which Media Foundation decodes everywhere. An HEVC file (a +//! recording made on a Mac) needs a decoder Windows does not always have, and the error from +//! [`Asset::start`] tells the user to install the HEVC Video Extensions or to record with H.264. +//! +//! # Tests +//! +//! The rule for which decoded frames answer a request is `Selector`, and its tests run on every +//! host. Nothing in this module runs without Windows and a recorded video: `doctor --probe decode` +//! measures first-frame time, decode speed and seeking there, and the checkpoint W3 in +//! `docs/dev/windows-checkpoints.md` is its first run. + +use super::Frame; +use super::Scale; +use super::Selector; +use crate::platform::media::pack; +use crate::platform::media::unpack; +use color_eyre::eyre; +use color_eyre::eyre::WrapErr; +use std::collections::VecDeque; +use std::path::Path; +use windows::Win32::Media::MediaFoundation::IMF2DBuffer; +use windows::Win32::Media::MediaFoundation::IMFAttributes; +use windows::Win32::Media::MediaFoundation::IMFSample; +use windows::Win32::Media::MediaFoundation::IMFSourceReader; +use windows::Win32::Media::MediaFoundation::MF_MT_DEFAULT_STRIDE; +use windows::Win32::Media::MediaFoundation::MF_MT_FRAME_SIZE; +use windows::Win32::Media::MediaFoundation::MF_MT_INTERLACE_MODE; +use windows::Win32::Media::MediaFoundation::MF_MT_MAJOR_TYPE; +use windows::Win32::Media::MediaFoundation::MF_MT_SUBTYPE; +use windows::Win32::Media::MediaFoundation::MF_PD_DURATION; +use windows::Win32::Media::MediaFoundation::MF_SOURCE_READER_ALL_STREAMS; +use windows::Win32::Media::MediaFoundation::MF_SOURCE_READER_ENABLE_VIDEO_PROCESSING; +use windows::Win32::Media::MediaFoundation::MF_SOURCE_READER_FIRST_VIDEO_STREAM; +use windows::Win32::Media::MediaFoundation::MF_SOURCE_READER_MEDIASOURCE; +use windows::Win32::Media::MediaFoundation::MF_SOURCE_READERF_ENDOFSTREAM; +use windows::Win32::Media::MediaFoundation::MFCreateAttributes; +use windows::Win32::Media::MediaFoundation::MFCreateMediaType; +use windows::Win32::Media::MediaFoundation::MFCreateSourceReaderFromURL; +use windows::Win32::Media::MediaFoundation::MFMediaType_Video; +use windows::Win32::Media::MediaFoundation::MFVideoFormat_HEVC; +use windows::Win32::Media::MediaFoundation::MFVideoFormat_RGB32; +use windows::Win32::Media::MediaFoundation::MFVideoInterlace_Progressive; +use windows::Win32::System::Com::StructuredStorage::PROPVARIANT; +use windows::Win32::System::Com::StructuredStorage::PropVariantToUInt64; +use windows::core::GUID; +use windows::core::Interface; +use windows::core::PCWSTR; + +/// The first video stream, as the Source Reader numbers streams. +const VIDEO: u32 = MF_SOURCE_READER_FIRST_VIDEO_STREAM.0 as u32; + +/// Media Foundation counts in hundred-nanosecond units. +const UNITS_PER_SECOND: f64 = 10_000_000.0; + +fn open_reader(path: &Path) -> eyre::Result { + crate::platform::media::start()?; + if !path.is_file() { + eyre::bail!("{} does not exist", path.display()); + } + + let wide: Vec = path + .to_string_lossy() + .encode_utf16() + .chain(Some(0)) + .collect(); + // SAFETY: attribute setters and a creation call with the arguments they document; `wide` + // outlives the call that reads it + unsafe { + let mut attributes: Option = None; + MFCreateAttributes(&raw mut attributes, 1)?; + let attributes = + attributes.ok_or_else(|| eyre::Report::msg("no Media Foundation attributes"))?; + // the video processor converts the decoded picture to the format asked for and resizes it + attributes.SetUINT32(&MF_SOURCE_READER_ENABLE_VIDEO_PROCESSING, 1)?; + + let reader = MFCreateSourceReaderFromURL(PCWSTR(wide.as_ptr()), &attributes) + .wrap_err_with(|| format!("Windows cannot open {} as a video", path.display()))?; + reader.SetStreamSelection(MF_SOURCE_READER_ALL_STREAMS.0 as u32, false)?; + reader.SetStreamSelection(VIDEO, true)?; + Ok(reader) + } +} + +/// An opened video file: its path, size and duration. +/// +/// Opening reads those facts and decodes nothing. Each [`Asset::start`] opens a new Source +/// Reader on the file and starts where it is told to. +pub struct Asset { + path: std::path::PathBuf, + width: u32, + height: u32, + duration_s: f64, + /// Whether the video is HEVC, which needs a decoder Windows does not always have. + hevc: bool, +} + +impl Asset { + /// Opens a video file. + /// + /// # Errors + /// + /// Returns an error when: + /// + /// - Windows Runtime or Media Foundation cannot be started; + /// - `path` is not a file that exists (` does not exist`); + /// - Windows cannot open the file as a video (`Windows cannot open as a video`); + /// - the file has no video track (` has no video track`), or its frame size cannot be + /// read. + pub fn open(path: &Path) -> eyre::Result { + let reader = open_reader(path)?; + + // SAFETY: property reads on a live reader + unsafe { + let native = reader + .GetNativeMediaType(VIDEO, 0) + .wrap_err_with(|| format!("{} has no video track", path.display()))?; + let (width, height) = unpack(native.GetUINT64(&MF_MT_FRAME_SIZE)?); + let hevc = native + .GetGUID(&MF_MT_SUBTYPE) + .is_ok_and(|subtype| subtype == MFVideoFormat_HEVC); + + let duration = reader + .GetPresentationAttribute(MF_SOURCE_READER_MEDIASOURCE.0 as u32, &MF_PD_DURATION) + .ok() + .and_then(|value| PropVariantToUInt64(&value).ok()) + .map_or(0.0, |units| units as f64 / UNITS_PER_SECOND); + + Ok(Self { + path: path.to_path_buf(), + width, + height, + duration_s: duration, + hevc, + }) + } + } + + /// Returns the size the video was recorded at, as `(width, height)` in pixels. + #[must_use] + pub fn size(&self) -> (u32, u32) { + (self.width, self.height) + } + + /// Returns the duration Media Foundation reports for the file, in seconds, or `0.0` if it + /// reports none (some truncated files). + #[must_use] + pub fn duration_s(&self) -> f64 { + self.duration_s + } + + /// Begins decoding at `from_s` seconds into the file, up to `to_s` if given (else the end). + /// + /// The first frame returned is the one on screen at `from_s`, which can be earlier than it. A + /// negative `from_s` is `0`. The end is `to_s`, or the file's + /// [`duration_s`](Asset::duration_s) if it is `None`, and is never before `from_s`: a file + /// whose duration is unknown (`0.0`) therefore decodes little more than the frame on screen at + /// `from_s` unless `to_s` is given. A frame at exactly the end is returned, and none after it. + /// + /// The Source Reader is asked for BGRA frames of the size `scale` asks for (see [`Scale`]). + /// When `from_s` is not `0` it seeks to the key frame at or before it, and the frames from + /// there up to `from_s` are decoded and passed over: a seek costs the frames between a key + /// frame and the time asked for. + /// + /// # Errors + /// + /// Returns an error when: + /// + /// - the file cannot be opened again (see [`Asset::open`]); + /// - the video is HEVC and Windows has no HEVC decoder (the message tells the user to install + /// the HEVC Video Extensions or to record with `--codec h264`); + /// - Windows cannot decode the video to pictures of that size and format; + /// - the seek to `from_s` fails (`could not seek in the video`). + pub fn start(&self, from_s: f64, to_s: Option, scale: Scale) -> eyre::Result { + let from = from_s.max(0.0); + let end = to_s.unwrap_or(self.duration_s).max(from); + let (width, height) = match scale { + Scale::Native => (self.width, self.height), + Scale::Width(wide) => { + let wide = wide.clamp(2, self.width.max(2)); + let high = (u64::from(wide) * u64::from(self.height) + / u64::from(self.width.max(1))) + .max(2); + // codecs like even sizes + (wide & !1, (high as u32) & !1) + } + }; + + let reader = open_reader(&self.path)?; + // SAFETY: setters on a media type just created and a reader just opened + unsafe { + let wanted = MFCreateMediaType()?; + wanted.SetGUID(&MF_MT_MAJOR_TYPE, &MFMediaType_Video)?; + wanted.SetGUID(&MF_MT_SUBTYPE, &MFVideoFormat_RGB32)?; + wanted.SetUINT32(&MF_MT_INTERLACE_MODE, MFVideoInterlace_Progressive.0 as u32)?; + wanted.SetUINT64(&MF_MT_FRAME_SIZE, pack(width, height))?; + // a positive stride is a top-down picture; Media Foundation's default for RGB is the + // other way up + wanted.SetUINT32(&MF_MT_DEFAULT_STRIDE, width * 4)?; + + if let Err(error) = reader.SetCurrentMediaType(VIDEO, None, &wanted) { + if self.hevc { + eyre::bail!( + "this video is HEVC and Windows has no HEVC decoder installed: install \ + the \"HEVC Video Extensions\" from the Microsoft Store, or record with \ + --codec h264 ({error})" + ); + } + return Err(error).wrap_err("Windows cannot decode this video to pictures"); + } + + if from > 0.0 { + // to the key frame at or before it; the frames up to `from` are decoded and passed + // over by the selector + reader + .SetCurrentPosition( + &GUID::zeroed(), + &PROPVARIANT::from((from * UNITS_PER_SECOND) as i64), + ) + .wrap_err("could not seek in the video")?; + } + } + + Ok(Reader { + reader, + selector: Selector::new(from, end), + ready: VecDeque::new(), + ended: false, + }) + } +} + +/// A started decode. Dropping it cancels the decode. +/// +/// Made only by [`Asset::start`]. It moves forward only: to look at another time, start another +/// decode. The frames it returns are the ones a `Selector` picks from what the Source Reader +/// decodes, so they begin with the frame on screen at the start of the range. +pub struct Reader { + reader: IMFSourceReader, + selector: Selector, + ready: VecDeque, + ended: bool, +} + +/// One decoded sample as a tightly packed BGRA frame. +fn frame_of(reader: &IMFSourceReader, sample: &IMFSample, time: i64) -> eyre::Result { + // SAFETY: the sample is live; the buffer is locked for exactly the copy and unlocked after + unsafe { + let current = reader.GetCurrentMediaType(VIDEO)?; + let (width, height) = unpack(current.GetUINT64(&MF_MT_FRAME_SIZE)?); + let (width, height) = (width as usize, height as usize); + + let buffer = sample.ConvertToContiguousBuffer()?; + let mut bgra = vec![0_u8; width * height * 4]; + + if let Ok(flat) = buffer.cast::() { + let (mut first, mut pitch) = (std::ptr::null_mut::(), 0_i32); + flat.Lock2D(&raw mut first, &raw mut pitch)?; + for row in 0..height { + // the pitch is negative for a picture stored bottom-up, and `first` is then the + // top row all the same + let source = first.offset(row as isize * pitch as isize); + std::ptr::copy_nonoverlapping( + source, + bgra.as_mut_ptr().add(row * width * 4), + width * 4, + ); + } + flat.Unlock2D()?; + } else { + let (mut data, mut length) = (std::ptr::null_mut::(), 0_u32); + buffer.Lock(&raw mut data, None, Some(&raw mut length))?; + let usable = (length as usize).min(bgra.len()); + std::ptr::copy_nonoverlapping(data, bgra.as_mut_ptr(), usable); + buffer.Unlock()?; + } + + // RGB32 leaves the fourth byte of each pixel undefined, and what draws the picture reads + // it as transparency + for pixel in bgra.as_chunks_mut::<4>().0 { + pixel[3] = 255; + } + + Ok(Frame { + pts_s: time as f64 / UNITS_PER_SECOND, + width: width as u32, + height: height as u32, + bgra, + }) + } +} + +impl Reader { + /// The next decoded frame of the file, or `None` at its end. + fn decode(&mut self) -> eyre::Result> { + loop { + let (mut flags, mut time) = (0_u32, 0_i64); + let mut sample: Option = None; + // SAFETY: the reader was set up by `start`; the out pointers are valid for the call + unsafe { + self.reader.ReadSample( + VIDEO, + 0, + None, + Some(&raw mut flags), + Some(&raw mut time), + Some(&raw mut sample), + ) + } + .wrap_err("could not read a frame")?; + + if flags & MF_SOURCE_READERF_ENDOFSTREAM.0 as u32 != 0 { + return Ok(None); + } + + // a tick with no picture, in a gap of the file: nothing yet + let Some(sample) = sample else { + continue; + }; + return frame_of(&self.reader, &sample, time).map(Some); + } + } + + /// Returns the next frame, or `None` at the end of the range. + /// + /// Frames come in order. Each is BGRA with the alpha byte set to opaque, because Media + /// Foundation leaves it undefined and what draws the picture reads it as transparency. + /// + /// # Errors + /// + /// Returns an error when the Source Reader fails to read a sample (`could not read a frame`), + /// or a decoded sample cannot be read as a picture. + pub fn next_frame(&mut self) -> eyre::Result> { + loop { + if let Some(frame) = self.ready.pop_front() { + return Ok(Some(frame)); + } + if self.selector.done() { + return Ok(None); + } + + let more = if self.ended { + Vec::new() + } else if let Some(frame) = self.decode()? { + self.selector.push(frame) + } else { + self.ended = true; + self.selector.finish() + }; + + if self.ended && more.is_empty() { + return Ok(None); + } + self.ready.extend(more); + } + } +} diff --git a/komorebi-recorder/src/av/mod.rs b/komorebi-recorder/src/av/mod.rs new file mode 100644 index 00000000..729c38ae --- /dev/null +++ b/komorebi-recorder/src/av/mod.rs @@ -0,0 +1,329 @@ +//! Decoding the recorded video. +//! +//! The recorder writes each display's video as a fragmented MP4 (see [`crate::capture`]). Reading +//! it back means asking the OS for the frames between two times, decoded to BGRA at a size of the +//! caller's choosing (the decoder scales, so a contact sheet never decodes more pixels than it +//! shows). This is the host axis of reading a session: the types below are the same everywhere and +//! the decoder behind them is the host's. +//! +//! | host | decoder | module | +//! |---|---|---| +//! | macOS | `AVFoundation`'s `AVAssetReader` | `reader` | +//! | Windows | Media Foundation's Source Reader | `mf` | +//! | any other | none: [`Asset::open`] fails | `unsupported` | +//! +//! The decoder modules are private. [`Asset`] and [`Reader`] are re-exports of the host's, and +//! [`supported`] says whether there is one. A host with no decoder can still read everything else +//! in a session. +//! +//! # Using it +//! +//! [`Asset::open`] a file, [`Asset::start`] a decode of a range of it at a [`Scale`], and call +//! [`Reader::next_frame`] until it returns `None`. The first frame is the one that was on screen at +//! the start of the range, and a recorded display only has frames where the screen changed, so a +//! range can hold few frames or one. +//! +//! ```rust,no_run +//! use komorebi_recorder::av::{Asset, Scale}; +//! use std::path::Path; +//! +//! let asset = Asset::open(Path::new("video-1.mp4"))?; +//! let (width, height) = asset.size(); +//! println!("{width}x{height}, {:.1} s", asset.duration_s()); +//! +//! // The frames of the second second of the file, decoded 320 pixels wide. +//! let mut reader = asset.start(1.0, Some(2.0), Scale::Width(320))?; +//! while let Some(frame) = reader.next_frame()? { +//! assert_eq!(frame.bgra.len(), frame.width as usize * frame.height as usize * 4); +//! println!("frame at {:.3} s", frame.pts_s); +//! } +//! # Ok::<(), Box>(()) +//! ``` +//! +//! The example needs a recorded video, so it is compiled and not run. What it does is covered on +//! macOS by the unit tests in `av::reader` (they build a video whose frames number themselves and +//! need `ffmpeg`), and on every host by the tests of the frame selection rule (see the private +//! `Selector`). Nothing exercises the Windows decoder without a Windows machine, where +//! `doctor --probe decode` measures it. +//! +//! # Why it is shaped this way +//! +//! `AVFoundation`'s objects are not `Send` and some calls raise an Objective-C exception instead of +//! returning an error when used in the wrong order, which would abort the process. The types here +//! make the wrong order impossible to write: the only way to a [`Reader`] is +//! [`Asset::start`], which configures everything before reading begins. A Media Foundation object +//! is likewise used only on the thread that made it, so on Windows too a decode is started and read +//! on one thread. +//! +//! # What this module does not do +//! +//! It does not decide which frames a recording means: choosing the moments to look at, comparing +//! frames and drawing contact sheets are done by `analysis` and `playback`, on top of this. It +//! decodes video only. It does not seek within a started decode: start another [`Reader`]. + +/// Returns `true` if and only if this host can decode the recorded video. +/// +/// That is macOS and Windows. Elsewhere [`Asset::open`] always fails. +/// +/// # Example +/// +/// ```rust +/// assert_eq!( +/// komorebi_recorder::av::supported(), +/// cfg!(any(target_os = "macos", windows)) +/// ); +/// ``` +#[must_use] +pub const fn supported() -> bool { + cfg!(any(target_os = "macos", windows)) +} + +#[cfg(windows)] +mod mf; +#[cfg(target_os = "macos")] +mod reader; +#[cfg(not(any(target_os = "macos", windows)))] +mod unsupported; + +#[cfg(windows)] +pub use mf::Asset; +#[cfg(windows)] +pub use mf::Reader; +#[cfg(target_os = "macos")] +pub use reader::Asset; +#[cfg(target_os = "macos")] +pub use reader::Reader; +#[cfg(not(any(target_os = "macos", windows)))] +pub use unsupported::Asset; +#[cfg(not(any(target_os = "macos", windows)))] +pub use unsupported::Reader; + +/// The size to decode frames at. +/// +/// The decoder does the scaling, so a request for small frames decodes fewer pixels than the +/// recording has. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Scale { + /// The size the video was recorded at ([`Asset::size`]). + Native, + /// This wide, keeping the aspect ratio. The width is limited to between 2 and the recorded + /// width (a frame is never scaled up), and the height follows from the aspect ratio. Both are + /// rounded down to an even number, as video codecs want, and are never less than 2. + Width(u32), +} + +/// One decoded frame, tightly packed BGRA (no row padding). +/// +/// The `bgra` field is `width * height * 4` bytes: the blue, green, red and alpha bytes of each +/// pixel, row by row from the top. +#[derive(Debug, Clone, PartialEq)] +pub struct Frame { + /// When the frame is shown, in seconds from the start of *this file*. The file's time zero is + /// its first frame, which was captured at [`DisplayResult::video_t0_mach_ns`]; the session's + /// video time of a file is offset from it by [`DisplayEntry::offset_s`]. + /// + /// [`DisplayResult::video_t0_mach_ns`]: crate::capture::DisplayResult::video_t0_mach_ns + /// [`DisplayEntry::offset_s`]: crate::session::DisplayEntry::offset_s + pub pts_s: f64, + /// The width of the picture in pixels. + pub width: u32, + /// The height of the picture in pixels. + pub height: u32, + /// The pixels: `width * height * 4` bytes, BGRA, top row first. Alpha is opaque on Windows, + /// where the decoder leaves it undefined. + pub bgra: Vec, +} + +/// Picks the frames a request for `from..=end` is answered with from the decoded frames of a file, +/// given in order. +/// +/// A decoder that seeks can only land on a key frame before the time asked for and decodes forward +/// from there, so the frames before `from` have to be told from the one that was on screen at +/// `from`, which is the last of them: it is the first frame the request gets, and it can be earlier +/// than `from`. Frames after `end` are not wanted and the first of them ends the request. +/// +/// The Windows decoder (`mf`) uses it, because Media Foundation's Source Reader seeks to the key +/// frame before the position and decodes forward. The macOS decoder (`reader`) does not: an +/// `AVAssetReader` given a time range starts at the frame on screen at its start. The rule is +/// host-independent, so its tests run on every host. +/// +/// # Protocol +/// +/// Create it with the request's `from` and `end` in seconds of the file's time, [`push`] each +/// decoded frame in order and collect what comes back, stop as soon as [`done`] is `true`, and if +/// the decoder runs out of frames first, collect [`finish`]. +/// +/// ```text +/// request 1.25..=2.0; the decoder lands on a key frame at 0.0 and decodes forward +/// +/// push(0.0) -> [] held: before the start, it may be the one on screen +/// push(0.5) -> [] held instead of 0.0 +/// push(1.0) -> [] held instead of 0.5 +/// push(1.5) -> [1.0, 1.5] the first frame at or after the start: the held 1.0 was on screen at +/// 1.25, so it is the request's first frame +/// push(2.0) -> [2.0] a frame exactly at the end is still wanted +/// push(2.5) -> [] later than the end: done() becomes true +/// ``` +/// +/// Edge cases, each covered by a unit test of this module: +/// +/// - a frame exactly at `from` is the first frame, and the held one before it is dropped; +/// - a request after the last frame of a still display is answered by [`finish`] with the last +/// frame before `from`, because nothing changed after it; +/// - a request before the first frame is answered from the first frame; +/// - an empty file answers with nothing. +/// +/// [`push`]: Selector::push +/// [`done`]: Selector::done +/// [`finish`]: Selector::finish +#[derive(Debug)] +#[cfg_attr(not(windows), allow(dead_code))] +pub(crate) struct Selector { + from_s: f64, + end_s: f64, + started: bool, + /// The most recent frame before `from`. + pending: Option, + done: bool, +} + +#[cfg_attr(not(windows), allow(dead_code))] +impl Selector { + /// Creates a selector for the frames shown from `from_s` to `end_s`, both in seconds of the + /// file's time. + pub(crate) fn new(from_s: f64, end_s: f64) -> Self { + Self { + from_s, + end_s, + started: false, + pending: None, + done: false, + } + } + + /// Returns `true` if and only if the request is answered in full: nothing more is wanted. + pub(crate) fn done(&self) -> bool { + self.done + } + + fn within(&mut self, frame: Frame) -> Vec { + if frame.pts_s > self.end_s { + self.done = true; + Vec::new() + } else { + vec![frame] + } + } + + /// Offers the next decoded frame and returns the frames that are ready to be handed out, in + /// order: none, one, or the held frame followed by this one. + /// + /// Returns nothing once the request is [done](Selector::done). + pub(crate) fn push(&mut self, frame: Frame) -> Vec { + if self.done { + return Vec::new(); + } + + if !self.started { + if frame.pts_s < self.from_s { + self.pending = Some(frame); + return Vec::new(); + } + + self.started = true; + // a frame exactly at `from` is the one on screen then; only when the first at or after + // it is later does the one before it still hold the screen + let before = self.pending.take().filter(|_| frame.pts_s > self.from_s); + let mut ready: Vec = before.into_iter().collect(); + ready.extend(self.within(frame)); + return ready; + } + + self.within(frame) + } + + /// Tells the selector that the file has no more frames, and returns what is left to hand out. + /// + /// If nothing later was found, the frame that was on screen at `from` is the answer (a still + /// display, where no frame follows). After a frame at or after `from` has been pushed, nothing + /// is left. + pub(crate) fn finish(&mut self) -> Vec { + self.done = true; + if self.started { + return Vec::new(); + } + + self.started = true; + self.pending.take().into_iter().collect() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn frame(pts_s: f64) -> Frame { + Frame { + pts_s, + width: 2, + height: 2, + bgra: vec![0; 16], + } + } + + fn times(frames: &[Frame]) -> Vec { + frames.iter().map(|frame| frame.pts_s).collect() + } + + fn run(from: f64, end: f64, decoded: &[f64]) -> Vec { + let mut selector = Selector::new(from, end); + let mut out = Vec::new(); + for pts in decoded { + out.extend(selector.push(frame(*pts))); + if selector.done() { + return times(&out); + } + } + out.extend(selector.finish()); + times(&out) + } + + #[test] + fn the_first_frame_is_the_one_that_was_on_screen_at_the_start() { + // decoding began at a key frame at 0.0; the request starts at 1.25 + assert_eq!( + run(1.25, 2.0, &[0.0, 0.5, 1.0, 1.5, 2.0, 2.5]), + [1.0, 1.5, 2.0] + ); + } + + #[test] + fn a_frame_exactly_at_the_start_is_the_first() { + assert_eq!(run(1.0, 1.5, &[0.0, 1.0, 1.5, 2.0]), [1.0, 1.5]); + } + + #[test] + fn nothing_after_the_end_is_returned_and_the_request_stops_there() { + let mut selector = Selector::new(0.0, 1.0); + assert_eq!(times(&selector.push(frame(0.5))), [0.5]); + assert_eq!(times(&selector.push(frame(1.5))), Vec::::new()); + assert!(selector.done()); + assert!(selector.push(frame(1.6)).is_empty()); + } + + #[test] + fn a_still_display_answers_with_the_last_frame_before_the_start() { + // the screen did not change after 1.0 (and the file ends): that is what was on screen at 5 + assert_eq!(run(5.0, 6.0, &[0.0, 1.0]), [1.0]); + } + + #[test] + fn a_request_before_the_first_frame_gets_the_first_frame() { + assert_eq!(run(0.0, 1.0, &[0.2, 0.6]), [0.2, 0.6]); + } + + #[test] + fn an_empty_file_answers_with_nothing() { + assert_eq!(run(0.0, 1.0, &[]), Vec::::new()); + } +} diff --git a/komorebi-recorder/src/av/reader.rs b/komorebi-recorder/src/av/reader.rs new file mode 100644 index 00000000..6a94a33e --- /dev/null +++ b/komorebi-recorder/src/av/reader.rs @@ -0,0 +1,489 @@ +//! Decoding recorded video on macOS with `AVFoundation`'s `AVAssetReader`. +//! +//! [`Asset`] is a file's first video track and the facts about it; [`Asset::start`] configures an +//! `AVAssetReader` completely (the time range, then an output asking for BGRA pixels at the size +//! wanted) before it calls `startReading`, which is the only order `AVFoundation` accepts without +//! raising an Objective-C exception. The decoder converts and scales, so a request for small frames +//! decodes fewer pixels than the recording has. The reader returns the frame that was on screen at +//! the start of the range as its first, so this module needs no frame selection (the +//! Windows decoder uses `Selector` for that). +//! +//! Measured on an Apple Silicon Mac with HEVC recordings of real displays, at time of writing +//! (2026-10-26): the first frame at a time arrives in 14 to 28 ms once warm, frames come at 250 to +//! 970 a second depending on the size, and the first open of a file takes about a second. The +//! ignored test `real_video_decode_speed` measures it on a recording named by `KOMOREBI_VIDEO`. +//! +//! The unit tests build a video whose frames are numbered (with `ffmpeg`; they skip +//! with a message without it) and read the number back out of the pixels, so seeking, ranges and +//! scaling are checked on what was decoded. + +use super::Frame; +use super::Scale; +use color_eyre::eyre; +use objc2::rc::Retained; +use objc2::runtime::AnyObject; +use objc2_av_foundation::AVAsset; +use objc2_av_foundation::AVAssetReader; +use objc2_av_foundation::AVAssetReaderOutput; +use objc2_av_foundation::AVAssetReaderStatus; +use objc2_av_foundation::AVAssetReaderTrackOutput; +use objc2_av_foundation::AVAssetTrack; +use objc2_av_foundation::AVMediaTypeVideo; +use objc2_core_media::CMTime; +use objc2_core_media::CMTimeRange; +use objc2_core_video::CVPixelBufferGetBaseAddress; +use objc2_core_video::CVPixelBufferGetBytesPerRow; +use objc2_core_video::CVPixelBufferGetHeight; +use objc2_core_video::CVPixelBufferGetWidth; +use objc2_core_video::CVPixelBufferLockBaseAddress; +use objc2_core_video::CVPixelBufferLockFlags; +use objc2_core_video::CVPixelBufferUnlockBaseAddress; +use objc2_foundation::NSDictionary; +use objc2_foundation::NSNumber; +use objc2_foundation::NSString; +use objc2_foundation::NSURL; +use std::path::Path; + +/// `kCVPixelFormatType_32BGRA` +const PIXEL_FORMAT_BGRA: usize = 0x4247_5241; + +/// The timescale used for the times handed to the decoder: finer than any frame rate. +const TIMESCALE: i32 = 60_000; + +/// An opened video file: its first video track, its size and its duration. +/// +/// Opening reads those facts and decodes nothing. Each [`Asset::start`] makes a new decode that +/// starts where it is told to. +pub struct Asset { + asset: Retained, + track: Retained, + width: u32, + height: u32, + duration_s: f64, +} + +impl Asset { + /// Opens a video file. + /// + /// # Errors + /// + /// Returns an error when: + /// + /// - `path` is not valid UTF-8; + /// - `path` is not a file that exists (` does not exist`); + /// - the file has no video track (` has no video track`), which is also what a file that + /// is not a video gives; + /// - `AVFoundation` does not provide the video media type. + /// + /// None of these aborts the process. + pub fn open(path: &Path) -> eyre::Result { + let text = path + .to_str() + .ok_or_else(|| eyre::Report::msg("the video path is not valid UTF-8"))?; + if !path.is_file() { + eyre::bail!("{} does not exist", path.display()); + } + + // SAFETY: plain object construction and property reads on live objects; the deprecated + // synchronous track accessor is fine for a local file + unsafe { + let url = NSURL::fileURLWithPath(&NSString::from_str(text)); + let asset = AVAsset::assetWithURL(&url); + + let media_type = AVMediaTypeVideo + .ok_or_else(|| eyre::Report::msg("AVMediaTypeVideo is unavailable"))?; + #[allow(deprecated)] + let tracks = asset.tracksWithMediaType(media_type); + let track = tracks.firstObject().ok_or_else(|| { + eyre::Report::msg(format!("{} has no video track", path.display())) + })?; + + let size = track.naturalSize(); + let duration = asset.duration(); + let duration_s = if duration.timescale > 0 && duration.value >= 0 { + duration.value as f64 / f64::from(duration.timescale) + } else { + 0.0 + }; + + Ok(Self { + asset, + track, + width: size.width.round().max(0.0) as u32, + height: size.height.round().max(0.0) as u32, + duration_s, + }) + } + } + + /// Returns the size the video was recorded at, as `(width, height)` in pixels. + #[must_use] + pub fn size(&self) -> (u32, u32) { + (self.width, self.height) + } + + /// Returns the duration `AVFoundation` reports for the file, in seconds, or `0.0` if it reports + /// none. + /// + /// For a fragmented MP4 that was never finalized (a recorder that was killed) this is the + /// playable duration, which is why recovery trusts it over `ffprobe`'s. + #[must_use] + pub fn duration_s(&self) -> f64 { + self.duration_s + } + + /// Begins decoding at `from_s` seconds into the file, up to `to_s` if given (else the end). + /// + /// The first frame returned is the one on screen at `from_s`, which can be earlier than it. A + /// negative `from_s` is `0`. The end is `to_s`, or the file's + /// [`duration_s`](Asset::duration_s) if it is `None`, and is never before `from_s`: a file + /// whose duration is unknown (`0.0`) therefore decodes little more than the frame on screen at + /// `from_s` unless `to_s` is given. The time range given to the decoder is counted in units of + /// 1/60000 of a second. A range past the end of the file is not an error: it gives at most the + /// last frame. + /// + /// Frames are converted to BGRA and scaled by the decoder to the size `scale` asks for (see + /// [`Scale`]). + /// + /// # Errors + /// + /// Returns an error when `AVFoundation` cannot make a reader for the file, rejects the output + /// configuration, or cannot start decoding. The messages carry the description `AVFoundation` + /// gives. + pub fn start(&self, from_s: f64, to_s: Option, scale: Scale) -> eyre::Result { + let from = from_s.max(0.0); + let end = to_s.unwrap_or(self.duration_s).max(from); + let (width, height) = match scale { + Scale::Native => (self.width, self.height), + Scale::Width(wide) => { + let wide = wide.clamp(2, self.width.max(2)); + let high = (u64::from(wide) * u64::from(self.height) / u64::from(self.width.max(1))) + .max(2) as u32; + // codecs like even sizes + (wide & !1, high & !1) + } + }; + + // SAFETY: every object is live; the reader is configured completely (time range, output) + // before `startReading`, which is the order AVFoundation requires + unsafe { + let reader = + AVAssetReader::assetReaderWithAsset_error(&self.asset).map_err(|error| { + eyre::Report::msg(format!( + "could not read the video: {}", + error.localizedDescription() + )) + })?; + + let start = CMTime::new((from * f64::from(TIMESCALE)).round() as i64, TIMESCALE); + let span = CMTime::new( + ((end - from) * f64::from(TIMESCALE)).ceil() as i64 + 1, + TIMESCALE, + ); + reader.setTimeRange(CMTimeRange { + start, + duration: span, + }); + + let format = NSNumber::new_usize(PIXEL_FORMAT_BGRA); + let wide = NSNumber::new_usize(width as usize); + let high = NSNumber::new_usize(height as usize); + let keys = [ + &*NSString::from_str("PixelFormatType"), + &*NSString::from_str("Width"), + &*NSString::from_str("Height"), + ]; + let values: [&AnyObject; 3] = [format.as_ref(), wide.as_ref(), high.as_ref()]; + let settings: Retained> = + NSDictionary::from_slices(&keys, &values); + + let output = AVAssetReaderTrackOutput::assetReaderTrackOutputWithTrack_outputSettings( + &self.track, + Some(&settings), + ); + // frames are only used once, in order + output.setAlwaysCopiesSampleData(false); + + let as_output: &AVAssetReaderOutput = &output; + if !reader.canAddOutput(as_output) { + eyre::bail!("the video decoder rejected the output configuration"); + } + reader.addOutput(as_output); + + if !reader.startReading() { + let why = reader.error().map_or_else( + || "unknown error".to_string(), + |error| error.localizedDescription().to_string(), + ); + eyre::bail!("could not start decoding the video: {why}"); + } + + Ok(Reader { reader, output }) + } + } +} + +/// A started decode. Dropping it cancels the decode. +/// +/// Made only by [`Asset::start`], so it has always been configured completely before reading +/// began. It moves forward only: to look at another time, start another decode. +pub struct Reader { + reader: Retained, + output: Retained, +} + +impl Reader { + /// Returns the next frame, or `None` at the end of the range. + /// + /// Frames come in order, and a sample with no picture is skipped. + /// + /// # Errors + /// + /// Returns an error when decoding fails (`decoding failed`, with `AVFoundation`'s description), + /// or the decoder returns a frame with no pixels. + pub fn next_frame(&mut self) -> eyre::Result> { + // SAFETY: the reader has been started (the only way to hold a `Reader`); the sample + // buffer and the pixel buffer it holds are live for as long as `sample` is + unsafe { + let Some(sample) = self.output.copyNextSampleBuffer() else { + return match self.reader.status() { + AVAssetReaderStatus::Failed => { + let why = self.reader.error().map_or_else( + || "unknown error".to_string(), + |error| error.localizedDescription().to_string(), + ); + Err(eyre::Report::msg(format!("decoding failed: {why}"))) + } + _ => Ok(None), + }; + }; + + let time = sample.presentation_time_stamp(); + let pts_s = if time.timescale > 0 { + time.value as f64 / f64::from(time.timescale) + } else { + 0.0 + }; + + let Some(image) = sample.image_buffer() else { + // a marker sample with no picture: skip it + return self.next_frame(); + }; + let pixels = image.as_ref(); + + CVPixelBufferLockBaseAddress(pixels, CVPixelBufferLockFlags::ReadOnly); + let width = CVPixelBufferGetWidth(pixels); + let height = CVPixelBufferGetHeight(pixels); + let stride = CVPixelBufferGetBytesPerRow(pixels); + let base = CVPixelBufferGetBaseAddress(pixels).cast::(); + + let row = width * 4; + let mut bgra = Vec::with_capacity(row * height); + if base.is_null() { + CVPixelBufferUnlockBaseAddress(pixels, CVPixelBufferLockFlags::ReadOnly); + eyre::bail!("the decoder returned a frame with no pixels"); + } + for y in 0..height { + let line = std::slice::from_raw_parts(base.add(y * stride), row); + bgra.extend_from_slice(line); + } + CVPixelBufferUnlockBaseAddress(pixels, CVPixelBufferLockFlags::ReadOnly); + + Ok(Some(Frame { + pts_s, + width: width as u32, + height: height as u32, + bgra, + })) + } + } +} + +impl Drop for Reader { + fn drop(&mut self) { + // SAFETY: cancelling a started reader is always allowed + unsafe { self.reader.cancelReading() }; + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::testkit; + + const FPS: u32 = 30; + const FRAMES: u32 = 90; + + fn video(name: &str) -> Option { + let dir = std::env::temp_dir().join(format!( + "komorebi-recorder-av-{name}-{}", + std::process::id() + )); + std::fs::create_dir_all(&dir).expect("dir"); + let path = dir.join("numbered.mp4"); + if testkit::write_numbered_video(&path, FRAMES, FPS) { + Some(path) + } else { + eprintln!("skipped: ffmpeg is not available to make a test video"); + None + } + } + + fn decode_all(asset: &Asset, from: f64, to: Option, scale: Scale) -> Vec<(f64, u32)> { + let mut reader = asset.start(from, to, scale).expect("start"); + let mut frames = Vec::new(); + while let Some(frame) = reader.next_frame().expect("frame") { + frames.push(( + frame.pts_s, + testkit::numbered_frame_index(&frame.bgra, frame.width, frame.height), + )); + } + frames + } + + #[test] + fn decoding_from_the_start_yields_every_frame_in_order() { + let Some(path) = video("all") else { return }; + let asset = Asset::open(&path).expect("open"); + + assert_eq!( + asset.size(), + (testkit::NUMBERED_WIDTH, testkit::NUMBERED_HEIGHT) + ); + assert!((asset.duration_s() - f64::from(FRAMES) / f64::from(FPS)).abs() < 0.1); + + let frames = decode_all(&asset, 0.0, None, Scale::Native); + assert_eq!(frames.len(), FRAMES as usize); + for (expected, (pts, index)) in frames.iter().enumerate() { + assert_eq!( + *index, expected as u32, + "frame {expected} was read as {index}" + ); + assert!( + (pts - expected as f64 / f64::from(FPS)).abs() < 0.002, + "frame {expected} at {pts}" + ); + } + } + + #[test] + fn starting_midway_begins_with_the_frame_on_screen_then() { + let Some(path) = video("seek") else { return }; + let asset = Asset::open(&path).expect("open"); + + // 1.0 s is frame 30; 1.02 s is still frame 30 (frame 31 appears at 1.0333) + for (from, expected) in [(1.0, 30), (1.02, 30), (2.5, 75), (0.0, 0)] { + let frames = decode_all(&asset, from, Some(from + 0.2), Scale::Native); + assert_eq!(frames[0].1, expected, "starting at {from} s"); + } + } + + #[test] + fn a_range_stops_where_it_was_asked_to() { + let Some(path) = video("range") else { return }; + let asset = Asset::open(&path).expect("open"); + + let frames = decode_all(&asset, 1.0, Some(1.5), Scale::Native); + let first = frames.first().expect("first").1; + let last = frames.last().expect("last").1; + assert_eq!(first, 30); + assert!((44..=46).contains(&last), "the range ended at frame {last}"); + } + + #[test] + fn the_decoder_scales() { + let Some(path) = video("scale") else { return }; + let asset = Asset::open(&path).expect("open"); + + let mut reader = asset + .start(1.0, Some(1.1), Scale::Width(64)) + .expect("start"); + let frame = reader.next_frame().expect("frame").expect("a frame"); + assert_eq!((frame.width, frame.height), (64, 32)); + assert_eq!(frame.bgra.len(), 64 * 32 * 4); + assert_eq!( + testkit::numbered_frame_index(&frame.bgra, frame.width, frame.height), + 30 + ); + } + + #[test] + fn reading_past_the_end_is_the_end_not_an_error() { + let Some(path) = video("past") else { return }; + let asset = Asset::open(&path).expect("open"); + + let frames = decode_all(&asset, 100.0, None, Scale::Native); + assert!(frames.len() <= 1, "{} frames past the end", frames.len()); + } + + #[test] + fn a_file_that_is_not_a_video_is_an_error_not_an_abort() { + let dir = + std::env::temp_dir().join(format!("komorebi-recorder-av-bad-{}", std::process::id())); + std::fs::create_dir_all(&dir).expect("dir"); + let text = dir.join("not-a-video.mp4"); + std::fs::write(&text, "this is not a video").expect("file"); + + assert!(Asset::open(&text).is_err()); + assert!(Asset::open(&dir.join("missing.mp4")).is_err()); + } + + /// Measures decoding a real recording: `KOMOREBI_VIDEO= cargo test ... -- --ignored + /// --nocapture`. Prints how long it takes to get the first frame at a time (the latency of + /// scrubbing) and how fast frames come once started (the rate of playing). + #[test] + #[ignore = "needs a real recording in KOMOREBI_VIDEO"] + fn real_video_decode_speed() { + use std::time::Instant; + + let Some(path) = std::env::var_os("KOMOREBI_VIDEO") else { + eprintln!("set KOMOREBI_VIDEO to a recorded video file"); + return; + }; + let opened = Instant::now(); + let asset = Asset::open(std::path::Path::new(&path)).expect("open"); + eprintln!( + "open {:?}: {}x{}, {:.1} s", + opened.elapsed(), + asset.size().0, + asset.size().1, + asset.duration_s() + ); + + for scale in [Scale::Width(320), Scale::Width(1200), Scale::Native] { + let mut first = Vec::new(); + for index in 0..8 { + let at = asset.duration_s() * (f64::from(index) + 0.5) / 8.0; + let started = Instant::now(); + let mut reader = asset.start(at, Some(at + 0.5), scale).expect("start"); + let frame = reader.next_frame().expect("frame").expect("a frame"); + first.push(started.elapsed().as_secs_f64() * 1000.0); + assert!( + frame.pts_s <= at + 0.05, + "the first frame is at or before the time asked" + ); + } + first.sort_by(f64::total_cmp); + + let started = Instant::now(); + let mut reader = asset.start(1.0, Some(3.0), scale).expect("start"); + let mut count = 0; + let mut size = (0, 0); + while let Some(frame) = reader.next_frame().expect("frame") { + count += 1; + size = (frame.width, frame.height); + } + let seconds = started.elapsed().as_secs_f64(); + eprintln!( + "{scale:?} ({}x{}): first frame median {:.0} ms (min {:.0}, max {:.0}); {count} frames in {:.2} s = {:.0} fps", + size.0, + size.1, + first[first.len() / 2], + first[0], + first[first.len() - 1], + seconds, + f64::from(count) / seconds + ); + } + } +} diff --git a/komorebi-recorder/src/av/unsupported.rs b/komorebi-recorder/src/av/unsupported.rs new file mode 100644 index 00000000..812f5eb3 --- /dev/null +++ b/komorebi-recorder/src/av/unsupported.rs @@ -0,0 +1,78 @@ +//! A host with no video decoder: opening a file says so. +//! +//! This module is built on every host that is neither macOS nor Windows, so that the investigation +//! commands (which only read a session directory) build and run there. [`Asset`] and [`Reader`] +//! have the same methods as the real ones and cannot be constructed: [`Asset::open`] always fails, +//! so nothing else here can ever be called. [`crate::av::supported`] is `false` on such a host, and +//! what needs video checks it first. Everything else in a session can still be read. + +use super::Frame; +use super::Scale; +use color_eyre::eyre; +use std::path::Path; + +fn unsupported() -> eyre::Report { + eyre::Report::msg(format!( + "decoding the recorded video is not implemented on {} yet; everything else in the \ + session can still be read", + std::env::consts::OS + )) +} + +/// An opened video file. There is no way to hold one on this host. +/// +/// The type has a field that cannot exist, so no value of it can: every method that takes `&self` +/// is unreachable. +pub struct Asset { + _never: std::convert::Infallible, +} + +impl Asset { + /// Always fails: this host has no video decoder. + /// + /// # Errors + /// + /// Always returns an error, whatever the path. The message explains that decoding the recorded + /// video is not implemented on this OS and that everything else in the session can still be + /// read. + pub fn open(_path: &Path) -> eyre::Result { + Err(unsupported()) + } + + /// Returns the size of the video. Unreachable: there is no [`Asset`] on this host. + #[must_use] + pub fn size(&self) -> (u32, u32) { + match self._never {} + } + + /// Returns the duration of the video. Unreachable: there is no [`Asset`] on this host. + #[must_use] + pub fn duration_s(&self) -> f64 { + match self._never {} + } + + /// Begins a decode. Unreachable: there is no [`Asset`] on this host. + /// + /// # Errors + /// + /// Never returns: there is no [`Asset`] to call it on. + pub fn start(&self, _from_s: f64, _to_s: Option, _scale: Scale) -> eyre::Result { + match self._never {} + } +} + +/// A started decode. There is no way to hold one on this host. +pub struct Reader { + _never: std::convert::Infallible, +} + +impl Reader { + /// Returns the next frame. Unreachable: there is no [`Reader`] on this host. + /// + /// # Errors + /// + /// Never returns: there is no [`Reader`] to call it on. + pub fn next_frame(&mut self) -> eyre::Result> { + match self._never {} + } +} diff --git a/komorebi-recorder/src/cli/mod.rs b/komorebi-recorder/src/cli/mod.rs index cfaef85b..1887b5ec 100644 --- a/komorebi-recorder/src/cli/mod.rs +++ b/komorebi-recorder/src/cli/mod.rs @@ -46,6 +46,11 @@ pub enum Command { Displays, /// Rebuild a session's timeline from its raw files, recovering it first if the recorder died Reindex(ReindexArgs), + /// Check the video-to-clock alignment against a rendered on-screen clock + Selftest(SelftestArgs), + /// The clock window used by `selftest` (internal) + #[command(hide = true)] + SelftestClock, } /// The arguments of `reindex`. @@ -55,6 +60,20 @@ pub struct ReindexArgs { pub session: PathBuf, } +/// The arguments of `selftest`. +#[derive(Args, Debug)] +pub struct SelftestArgs { + /// Seconds to record + #[arg(long, default_value_t = 5.0)] + pub seconds: f64, + /// Largest allowed disagreement between the video and the clocks, in milliseconds + #[arg(long, default_value_t = 100.0)] + pub tolerance_ms: f64, + /// Session directory for the self-test recording + #[arg(long)] + pub out: Option, +} + impl Command { /// Whether this command was asked for JSON, which also decides how a failure is reported. #[must_use] @@ -98,6 +117,8 @@ pub fn run(command: Command) -> eyre::Result<()> { Command::Record(args) => record::record(&args), Command::Displays => record::displays(), Command::Reindex(args) => reindex(&args), + Command::Selftest(args) => crate::selftest::run(&args), + Command::SelftestClock => crate::selftest::run_clock(), } } diff --git a/komorebi-recorder/src/lib.rs b/komorebi-recorder/src/lib.rs index 2849d765..6e16b5c1 100644 --- a/komorebi-recorder/src/lib.rs +++ b/komorebi-recorder/src/lib.rs @@ -56,6 +56,7 @@ //! | [`recovery`] | rebuilding a session whose recorder died | //! | [`cli`] | the command line | //! | [`app`] | the recorder window | +//! | [`av`] and [`selftest`] | decoding the recorded video and checking it against the clocks | //! //! # What is not supported //! @@ -71,6 +72,7 @@ pub mod analysis; pub mod annotations; pub mod app; +pub mod av; pub mod capture; pub mod cli; pub mod clock; @@ -82,6 +84,7 @@ pub mod platform; pub mod power; pub mod recorder; pub mod recovery; +pub mod selftest; pub mod session; pub mod sources; pub mod system_state; diff --git a/komorebi-recorder/src/platform/win/capture.rs b/komorebi-recorder/src/platform/win/capture.rs index 099563dc..be50c547 100644 --- a/komorebi-recorder/src/platform/win/capture.rs +++ b/komorebi-recorder/src/platform/win/capture.rs @@ -214,12 +214,12 @@ pub fn main_display() -> eyre::Result { /// timer and can lag behind (or run ahead of) what actually reached the video. A file whose /// duration it cannot read (some truncated ones), or that is not longer than zero, is `None`, and /// recovery says it could not tell. On macOS the duration comes from `AVFoundation`. -/// -/// Media Foundation's reader is not part of the build yet, so this is always `None`. #[must_use] pub fn media_duration_s(path: &Path) -> Option { - let _ = path; - None + crate::av::Asset::open(path) + .ok() + .map(|asset| asset.duration_s()) + .filter(|seconds| *seconds > 0.0) } /// The [`CaptureBackend`](crate::capture::CaptureBackend) of this host: Windows.Graphics.Capture. diff --git a/komorebi-recorder/src/selftest.rs b/komorebi-recorder/src/selftest.rs new file mode 100644 index 00000000..23b7166c --- /dev/null +++ b/komorebi-recorder/src/selftest.rs @@ -0,0 +1,844 @@ +//! The sync self-test: prove that "video second N" really is "clock time N". +//! +//! A child process (so the recorder does not exclude it from the capture along with its own +//! windows) draws a large live clock plus a strip of 40 black/white cells that binary-encodes the +//! host clock in milliseconds at the moment each frame was painted. The parent records the screen +//! headlessly, then for several video timestamps decodes the strip out of the extracted frame and +//! compares it with the time the session's manifest predicts for that video second. Nothing is +//! read by eye or OCR. +//! +//! What the comparison includes beyond the recorder's own accuracy: the clock is painted a +//! little before the compositor shows it and SCK frames are rate-limited, so the visible clock +//! trails the video time by one to a few frame intervals. That is why the default tolerance is +//! 100 ms and the mean bias is reported separately. +//! +//! # The pieces +//! +//! | Item | Is | +//! |---|---| +//! | [`run`] | the `selftest` command: records, decodes, compares, and stores the verdict in the manifest | +//! | [`run_clock`] | the child process: the window that paints the clock and the strip (the hidden `selftest-clock` command) | +//! | [`encode_cells`], [`decode_cells`] | the strip: a clock value to cell colors, and the average luminance of each cell back to a clock value | +//! | [`strip_filter`], [`strip_units`] | where the strip is in a display's video, as an `ffmpeg` crop | +//! | [`wrapped_difference_ms`], [`evaluate`] | the comparison, and the verdict ([`Reading`] in, [`Calibration`] out) | +//! +//! The pure parts (the strip, the geometry, the comparison) are the same on both hosts and are +//! tested without a screen. [`run`] and [`run_clock`] need a screen, a window system and `ffmpeg`, +//! so no automated test covers them: the `selftest` command is the check, and +//! `docs/dev/windows-checkpoints.md` (W2) says what to expect of it on Windows. +//! +//! # The strip +//! +//! The strip is 40 cells, one for each bit of the master clock in milliseconds, most significant +//! first, white for 1 and black for 0, of the value taken modulo 2^40 (a little under 35 years of +//! milliseconds, so it does not wrap in practice). It is painted at the same place in the clock +//! window every frame, and [`strip_filter`] says where that is in a display's video. The window +//! also shows the master clock and the wall clock as text, for a person looking at the video. +//! +//! # Hosts +//! +//! The recording itself is the host's capture. The geometry differs: on macOS the window's +//! position and the display's frame are both in points, and the video is a number of pixels to the +//! point; on Windows the display's frame is physical pixels and the window's position is still in +//! points. [`strip_units`] brings them to common units. +//! +//! # Example +//! +//! The strip carries a clock reading, and the difference between what the video shows and what the +//! manifest predicts is the error: +//! +//! ```rust +//! use komorebi_recorder::selftest::{decode_cells, encode_cells, evaluate, Reading}; +//! +//! // the clock at the moment a frame was painted, in ms: encoded as cells, then read back from +//! // the average luminance of each cell in the video +//! let painted_ms = 10_000; +//! let luminance: Vec = encode_cells(painted_ms) +//! .iter() +//! .map(|white| if *white { 255 } else { 0 }) +//! .collect(); +//! let observed_ms = decode_cells(&luminance).expect("a legible strip"); +//! assert_eq!(observed_ms, painted_ms); +//! +//! // the manifest predicted the video second to be at 10 030 ms: the video is 30 ms behind +//! let verdict = evaluate( +//! &[Reading { video_s: 1.0, predicted_ms: 10_030, observed_ms }], +//! 100.0, +//! ); +//! assert!(verdict.passed); +//! assert_eq!(verdict.samples[0].error_ms, -30.0); +//! ``` + +use crate::cli::SelftestArgs; +use crate::clock; +use crate::frames; +use crate::platform::capture; +use crate::recorder; +use crate::recorder::Live; +use crate::recorder::OsLogOptions; +use crate::recorder::RecordOptions; +use crate::session::Calibration; +use crate::session::CalibrationSample; +use crate::session::Manifest; +use crate::session::Session; +use crate::timecode; +use color_eyre::eyre; +use color_eyre::eyre::WrapErr; +use eframe::egui; +use std::io::BufRead; +use std::io::BufReader; +use std::io::Write; +use std::process::Command; +use std::process::Stdio; +use std::time::Duration; +use std::time::Instant; + +/// Number of cells in the strip, i.e. bits of the encoded millisecond clock. +const CELLS: usize = 40; +const CELL_WIDTH: f32 = 24.0; +const CELL_HEIGHT: f32 = 60.0; +/// Where the strip sits inside the clock window, in points. +const STRIP_X: f32 = 20.0; +const STRIP_Y: f32 = 110.0; +const WINDOW_SIZE: [f32; 2] = [1000.0, 260.0]; +const WINDOW_POSITION: [f32; 2] = [80.0, 80.0]; + +const WRAP_MS: u64 = 1 << CELLS; + +/// Returns the cell colors (true = white) for a clock value in milliseconds, most significant +/// first. +/// +/// There are 40 cells, so the value is taken modulo 2^40: bits above the lowest 40 are not shown. +/// +/// # Example +/// +/// ```rust +/// use komorebi_recorder::selftest::encode_cells; +/// +/// // 5 is 101 in binary: the last three cells, with the rest black +/// let cells = encode_cells(5); +/// assert_eq!(cells.len(), 40); +/// assert!(cells[..37].iter().all(|white| !white)); +/// assert_eq!(cells[37..], [true, false, true]); +/// ``` +#[must_use] +pub fn encode_cells(ms: u64) -> [bool; CELLS] { + let mut cells = [false; CELLS]; + for (index, cell) in cells.iter_mut().enumerate() { + *cell = (ms >> (CELLS - 1 - index)) & 1 == 1; + } + cells +} + +/// Recovers the clock value from the average luminance of each cell, 0 (black) to 255 (white). +/// +/// A cell at 128 or above is white. Returns `None` if there are not exactly 40 values, or if more +/// than 3 of them are neither near black nor near white (from 64 up to, but not including, 192), +/// which is what the average of a crop that missed the strip looks like. Cells that are soft but +/// still clearly one or the other, as anti-aliasing and video compression make them, decode. +/// +/// # Example +/// +/// ```rust +/// use komorebi_recorder::selftest::{decode_cells, encode_cells}; +/// +/// // a clock value survives encoding and reading back from the cells' luminance +/// let ms = 1_210_934_273; +/// let sharp: Vec = encode_cells(ms).iter().map(|white| if *white { 255 } else { 0 }).collect(); +/// assert_eq!(decode_cells(&sharp), Some(ms)); +/// +/// // soft edges still decode +/// let soft: Vec = sharp.iter().map(|level| if *level == 255 { 230 } else { 12 }).collect(); +/// assert_eq!(decode_cells(&soft), Some(ms)); +/// +/// // flat gray is a crop that missed the strip, and a short crop is not a strip +/// assert_eq!(decode_cells(&[128; 40]), None); +/// assert_eq!(decode_cells(&[0; 10]), None); +/// ``` +#[must_use] +pub fn decode_cells(luminance: &[u8]) -> Option { + if luminance.len() != CELLS { + return None; + } + + // a legible strip is overwhelmingly near-black or near-white; a mid-grey average means the + // crop missed the strip + let ambiguous = luminance + .iter() + .filter(|value| (64..192).contains(*value)) + .count(); + if ambiguous > 3 { + return None; + } + + Some(luminance.iter().fold(0_u64, |value, level| { + (value << 1) | u64::from(*level >= 128) + })) +} + +/// Returns `observed - predicted` in milliseconds, accounting for the strip's wrap-around. +/// +/// The strip holds a clock value modulo 2^40 milliseconds, so two values on either side of a wrap +/// are close, however far apart they are as numbers. The result is the shorter way round: within +/// half the wrap, in either direction. +/// +/// # Example +/// +/// ```rust +/// use komorebi_recorder::selftest::wrapped_difference_ms; +/// +/// assert_eq!(wrapped_difference_ms(1_050, 1_000), 50); +/// assert_eq!(wrapped_difference_ms(950, 1_000), -50); +/// +/// // the strip wrapped between the prediction and the observation +/// let wrap = 1_u64 << 40; +/// assert_eq!(wrapped_difference_ms(5, wrap - 5), 10); +/// assert_eq!(wrapped_difference_ms(wrap - 5, 5), -10); +/// ``` +#[must_use] +pub fn wrapped_difference_ms(observed: u64, predicted: u64) -> i64 { + let wrapped = (observed.wrapping_sub(predicted)) % WRAP_MS; + if wrapped >= WRAP_MS / 2 { + wrapped as i64 - WRAP_MS as i64 + } else { + wrapped as i64 + } +} + +/// Returns the ffmpeg `-vf` expression that crops the strip out of a display's video and averages +/// each cell down to one gray pixel. +/// +/// `window_origin` is the clock window's top left corner and `display_origin` the display's, in +/// the same units, and `scale` is how many pixels of video one of those units is. The crop is the +/// strip's position in the window (20, 110) and its size (40 cells of 24 by 60) in those units, +/// multiplied by `scale`, and cut down to the middle half of the cells' height so that the edges +/// of the strip do not blur into the average. The cells are then scaled to 40 by 1 pixels with area +/// averaging. +/// +/// # Example +/// +/// ```rust +/// use komorebi_recorder::selftest::strip_filter; +/// +/// // the window at (80, 80) on a display at the origin, one pixel to the unit +/// assert_eq!( +/// strip_filter((80.0, 80.0), (0.0, 0.0), 1.0), +/// "crop=960:30:100:205,scale=40:1:flags=area" +/// ); +/// +/// // the same window on a display to the left, two pixels to the unit +/// assert_eq!( +/// strip_filter((-1720.0, 80.0), (-1800.0, 0.0), 2.0), +/// "crop=1920:60:200:410,scale=40:1:flags=area" +/// ); +/// ``` +#[must_use] +pub fn strip_filter(window_origin: (f64, f64), display_origin: (f64, f64), scale: f64) -> String { + let x = ((window_origin.0 - display_origin.0) + f64::from(STRIP_X)) * scale; + let y = ((window_origin.1 - display_origin.1) + f64::from(STRIP_Y)) * scale; + let width = f64::from(CELL_WIDTH) * CELLS as f64 * scale; + // sample the middle of the cells vertically + let height = f64::from(CELL_HEIGHT) * 0.5 * scale; + let y = y + f64::from(CELL_HEIGHT) * 0.25 * scale; + + format!( + "crop={}:{}:{}:{},scale={CELLS}:1:flags=area", + width.round() as i64, + height.round().max(1.0) as i64, + x.round() as i64, + y.round() as i64, + ) +} + +/// Where the strip is in a display's video, as `strip_filter` wants it: the window's and the +/// display's origins in the same units, and how many pixels of video one of those units is. +/// +/// On a Mac the window (egui's points) and the display (CoreGraphics' points) share units and the +/// video is `video_scale` pixels to the point. On Windows the video is the display's physical +/// pixels and the window is still in points, `pixels_per_point` of which make a pixel, so the +/// display's origin is brought into points and the unit is the pixels-per-point. +/// +/// `display_frame_is_pixels` says which: true where the display's frame is in physical pixels +/// (Windows). A `pixels_per_point` below 0.1 is taken as 0.1. +/// +/// # Example +/// +/// ```rust +/// use komorebi_recorder::selftest::{strip_filter, strip_units}; +/// +/// // a Mac: both in points, the video 2 pixels to the point +/// assert_eq!(strip_units((100.0, 50.0), 2.0, 2.0, false), ((100.0, 50.0), 2.0)); +/// +/// // Windows at 150 %: the display at 1920 px is at 1280 points, and a point is 1.5 pixels +/// let ((x, y), unit) = strip_units((1920.0, 0.0), 1.0, 1.5, true); +/// assert_eq!((x, y, unit), (1280.0, 0.0, 1.5)); +/// +/// // so a window 80 points into that display puts its strip 150 pixels in +/// let filter = strip_filter((1280.0 + 80.0, 80.0), (x, y), unit); +/// assert!(filter.starts_with("crop=1440:45:150:")); +/// ``` +#[must_use] +pub fn strip_units( + display_frame_origin: (f64, f64), + video_scale: f64, + pixels_per_point: f64, + display_frame_is_pixels: bool, +) -> ((f64, f64), f64) { + if display_frame_is_pixels { + let ppp = pixels_per_point.max(0.1); + ( + (display_frame_origin.0 / ppp, display_frame_origin.1 / ppp), + ppp, + ) + } else { + (display_frame_origin, video_scale) + } +} + +/// The child process: a borderless, untitled (so komorebi leaves it alone) window that paints +/// the clock as fast as it can. +/// +/// The window is 1000 by 260 points, always on top, at (80, 80). It paints the master clock and the +/// wall clock as text, and the strip (see the [module docs](self#the-strip)) from a clock reading +/// taken as each frame's content is decided. It prints a line `RECT +/// ` on its standard output whenever its inner rectangle changes, which is how +/// [`run`] finds it. +/// +/// This blocks until the window is closed, or the parent kills it. It exits the process after 120 +/// seconds, in case the parent died without killing it. +/// +/// # Errors +/// +/// Returns an error if the window system cannot create the window (`the clock window failed`). +/// +/// This opens a window, so it has no example and no automated test; it runs when [`run`] starts it +/// as the hidden `selftest-clock` command. +pub fn run_clock() -> eyre::Result<()> { + let options = eframe::NativeOptions { + viewport: egui::ViewportBuilder::default() + .with_title("") + .with_decorations(false) + .with_position(WINDOW_POSITION) + .with_inner_size(WINDOW_SIZE) + .with_always_on_top() + .with_active(false), + ..Default::default() + }; + + eframe::run_native( + "komorebi-recorder-clock", + options, + Box::new(|_| Ok(Box::new(ClockApp::default()))), + ) + .map_err(|error| eyre::Report::msg(format!("the clock window failed: {error}"))) +} + +#[derive(Default)] +struct ClockApp { + last_rect: Option, + started: Option, +} + +impl eframe::App for ClockApp { + fn update(&mut self, context: &egui::Context, _frame: &mut eframe::Frame) { + let started = *self.started.get_or_insert_with(Instant::now); + // never linger if the parent dies without killing us + if started.elapsed() > Duration::from_secs(120) { + std::process::exit(0); + } + + // read the clock first: this is the moment the frame's content is decided + let mach_ns = clock::mono_now_ns(); + let wall_us = clock::wall_now_us(); + let ms = (mach_ns / 1_000_000) % WRAP_MS; + + if let Some(rect) = context.input(|input| input.viewport().inner_rect) + && self.last_rect != Some(rect) + { + self.last_rect = Some(rect); + println!( + "RECT {} {} {} {} {}", + rect.min.x, + rect.min.y, + rect.width(), + rect.height(), + context.pixels_per_point() + ); + let _ = std::io::stdout().flush(); + } + + egui::CentralPanel::default() + .frame(egui::Frame::NONE.fill(egui::Color32::from_gray(40))) + .show(context, |ui| { + let painter = ui.painter(); + painter.text( + egui::pos2(20.0, 14.0), + egui::Align2::LEFT_TOP, + format!("mach {mach_ns} ns"), + egui::FontId::monospace(30.0), + egui::Color32::WHITE, + ); + painter.text( + egui::pos2(20.0, 56.0), + egui::Align2::LEFT_TOP, + crate::timeline::format_wall_us(wall_us).unwrap_or_default(), + egui::FontId::monospace(30.0), + egui::Color32::LIGHT_GREEN, + ); + + for (index, white) in encode_cells(ms).into_iter().enumerate() { + let cell = egui::Rect::from_min_size( + egui::pos2(STRIP_X + index as f32 * CELL_WIDTH, STRIP_Y), + egui::vec2(CELL_WIDTH, CELL_HEIGHT), + ); + painter.rect_filled( + cell, + 0.0, + if white { + egui::Color32::WHITE + } else { + egui::Color32::BLACK + }, + ); + } + }); + + context.request_repaint(); + } +} + +/// One decoded sample, before it is turned into a [`CalibrationSample`]. +#[derive(Debug, Clone, PartialEq)] +pub struct Reading { + /// The video time the frame was taken at, in seconds. + pub video_s: f64, + /// The clock value the manifest predicts for that video time, in milliseconds on the master + /// clock modulo 2^40. + pub predicted_ms: u64, + /// The clock value read from the strip in the frame, in the same units. + pub observed_ms: u64, +} + +/// Turns readings into a calibration verdict. +/// +/// Each reading's error is the observed clock minus the predicted one, in milliseconds +/// ([`wrapped_difference_ms`]), so a negative error means the video shows a clock that is behind +/// what the manifest predicts. The verdict passes if there is at least one reading and the largest +/// error, in either direction, is no more than `tolerance_ms`. The verdict stores the readings as +/// samples, with the times in nanoseconds on the master clock. +/// +/// # Example +/// +/// ```rust +/// use komorebi_recorder::selftest::{evaluate, Reading}; +/// +/// let readings = vec![ +/// Reading { video_s: 1.0, predicted_ms: 10_000, observed_ms: 9_970 }, +/// Reading { video_s: 1.5, predicted_ms: 10_500, observed_ms: 10_460 }, +/// ]; +/// +/// let verdict = evaluate(&readings, 100.0); +/// assert!(verdict.passed); +/// assert_eq!(verdict.max_abs_error_ms, 40.0); +/// assert_eq!(verdict.samples[0].error_ms, -30.0); +/// +/// // the same readings against a tighter tolerance +/// assert!(!evaluate(&readings, 35.0).passed); +/// +/// // no readings is not a pass +/// assert!(!evaluate(&[], 100.0).passed); +/// ``` +#[must_use] +pub fn evaluate(readings: &[Reading], tolerance_ms: f64) -> Calibration { + let samples: Vec = readings + .iter() + .map(|reading| { + let error_ms = wrapped_difference_ms(reading.observed_ms, reading.predicted_ms) as f64; + CalibrationSample { + video_s: reading.video_s, + predicted_mach_ns: reading.predicted_ms * 1_000_000, + observed_mach_ns: reading.observed_ms * 1_000_000, + error_ms, + } + }) + .collect(); + + let max_abs_error_ms = samples + .iter() + .map(|sample| sample.error_ms.abs()) + .fold(0.0, f64::max); + + Calibration { + method: "on-screen binary clock strip decoded from extracted video frames".to_string(), + passed: !samples.is_empty() && max_abs_error_ms <= tolerance_ms, + max_abs_error_ms, + tolerance_ms, + samples, + } +} + +/// Decode the strip from one frame of one display file. +fn read_strip( + session: &Session, + file: &str, + file_s: f64, + filter: &str, +) -> eyre::Result> { + let output = Command::new("ffmpeg") + .args(["-v", "error", "-ss", &format!("{file_s:.3}"), "-i"]) + .arg(session.path(file)) + .args(["-frames:v", "1", "-vf", filter]) + .args(["-pix_fmt", "gray", "-f", "rawvideo", "-"]) + .output() + .wrap_err("could not run ffmpeg")?; + + if !output.status.success() { + eyre::bail!( + "ffmpeg failed at {}: {}", + timecode::format(file_s), + String::from_utf8_lossy(&output.stderr).trim() + ); + } + + Ok(decode_cells(&output.stdout)) +} + +/// `[x, y, width, height, pixels_per_point]`, the window in points. +type WindowReport = [f64; 5]; + +fn spawn_clock() -> eyre::Result<( + std::process::Child, + crossbeam_channel::Receiver, +)> { + let mut child = Command::new(std::env::current_exe()?) + .arg("selftest-clock") + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .spawn() + .wrap_err("could not start the clock window")?; + + let stdout = child + .stdout + .take() + .ok_or_else(|| eyre::Report::msg("the clock window has no stdout"))?; + + let (tx, rx) = crossbeam_channel::unbounded(); + std::thread::spawn(move || { + for line in BufReader::new(stdout).lines().map_while(Result::ok) { + let numbers: Vec = line + .strip_prefix("RECT ") + .map(|rest| { + rest.split_whitespace() + .filter_map(|n| n.parse().ok()) + .collect() + }) + .unwrap_or_default(); + + match numbers[..] { + [x, y, w, h, ppp] => { + let _ = tx.send([x, y, w, h, ppp]); + } + // an older clock window did not say + [x, y, w, h] => { + let _ = tx.send([x, y, w, h, 1.0]); + } + _ => {} + } + } + }); + + Ok((child, rx)) +} + +/// Runs the self-test: records the screen, reads the clock strip out of the video, and compares +/// it with what the session's manifest predicts. +/// +/// It starts the clock window as a child process ([`run_clock`], through the hidden +/// `selftest-clock` command of this same executable) on the main display, waits for it to say where +/// it is, and records that display alone for `args.seconds` seconds with no komorebi and no OS +/// trace. Then, for every half second of video from 1 s to half a second before the end, it asks +/// `ffmpeg` for that frame, reads the clock out of the strip, and compares it with the clock the +/// manifest predicts for that video second. A moment at which the strip cannot be read is not a +/// reading, and is noted in the manifest's warnings. The verdict ([`evaluate`]) is stored in the +/// manifest as `calibration`, a table of the errors is printed, and the session directory is +/// printed last. +/// +/// The recording is kept: `args.out` is where it goes, and by default it is next to the other +/// recordings. What is printed is more than a pass or a fail: the error of each sample, and their +/// mean, the bias, which is reported apart from the worst error because the clock is painted +/// slightly before it is shown. +/// +/// # Errors +/// +/// Returns an error if +/// +/// * `ffmpeg` is not on `PATH`; +/// * the OS has not let this process capture the screen (on macOS, the Screen Recording +/// permission), or the main display cannot be found; +/// * the clock window cannot be started, or does not report its position within 20 seconds; +/// * the recording fails, for any reason `record` would fail; +/// * no video was recorded for the main display; +/// * the clock cannot be found in any frame (is the clock window covered, or moved?): a frame is +/// saved under the session's `frames/` directory for a look; +/// * the manifest cannot be written; +/// * the test **fails**: the video disagrees with the clocks by more than `args.tolerance_ms`. The +/// verdict is in the manifest by then. +/// +/// # Example +/// +/// ```rust,no_run +/// use komorebi_recorder::cli::SelftestArgs; +/// use komorebi_recorder::selftest::run; +/// +/// // record for 8 seconds, and allow the video to disagree with the clocks by up to 100 ms +/// run(&SelftestArgs { seconds: 8.0, tolerance_ms: 100.0, out: None })?; +/// # Ok::<(), color_eyre::eyre::Report>(()) +/// ``` +/// +/// This needs a screen, a window system and `ffmpeg`, so it is not run, and no automated test +/// covers it: `komorebi-recorder selftest` is the check. +pub fn run(args: &SelftestArgs) -> eyre::Result<()> { + if !frames::ffmpeg_available() { + eyre::bail!("the self-test decodes frames with ffmpeg, which was not found on PATH"); + } + + capture::ensure_permission()?; + + let display = capture::main_display()?; + let main_id = display.id; + + eprintln!("starting the clock window on display {main_id}..."); + let (mut child, rect_rx) = spawn_clock()?; + + let first = rect_rx + .recv_timeout(Duration::from_secs(20)) + .map_err(|_| eyre::Report::msg("the clock window never reported its position"))?; + + // let the window settle (and be shown) before trusting its position + std::thread::sleep(Duration::from_millis(1500)); + let mut rect = first; + while let Ok(latest) = rect_rx.try_recv() { + rect = latest; + } + + let options = RecordOptions { + out: args.out.clone(), + displays: vec![main_id], + oslog: OsLogOptions { + enabled: false, + ..OsLogOptions::default() + }, + duration: Some(Duration::from_secs_f64(args.seconds)), + no_komorebi: true, + ..RecordOptions::default() + }; + + eprintln!("recording {:.0}s...", args.seconds); + let live = Live::new(); + let recorded = recorder::record(&options, &live); + let _ = child.kill(); + let _ = child.wait(); + let dir = recorded?; + + let session = Session::open(&dir)?; + let mut manifest = session.manifest()?; + let entry = manifest + .displays + .iter() + .find(|entry| entry.result.display.id == main_id && entry.result.frames > 0) + .cloned() + .ok_or_else(|| eyre::Report::msg("no video was recorded for the main display"))?; + + let (display_origin, video_per_unit) = strip_units( + (display.frame[0], display.frame[1]), + entry.result.scale, + rect[4], + cfg!(windows), + ); + let filter = strip_filter((rect[0], rect[1]), display_origin, video_per_unit); + + let (_, duration) = manifest.video_span(); + let mut readings = Vec::new(); + let mut undecodable = Vec::new(); + let mut video_s = 1.0; + while video_s <= duration - 0.5 { + let file_s = video_s - entry.offset_s; + let predicted_ms = manifest.mach_at_video_seconds(video_s) / 1_000_000 % WRAP_MS; + match read_strip(&session, &entry.result.file, file_s, &filter)? { + Some(observed_ms) => readings.push(Reading { + video_s, + predicted_ms, + observed_ms, + }), + None => undecodable.push(video_s), + } + video_s += 0.5; + } + + if readings.is_empty() { + // leave a frame behind so the problem can be looked at + let _ = frames::extract(&session, &manifest, &[1.0], Some(main_id)); + eyre::bail!( + "could not find the clock in the video (is the clock window covered, or was it moved? \ + window reported at {rect:?}); a frame was saved under {}", + session.path("frames").display() + ); + } + + let calibration = evaluate(&readings, args.tolerance_ms); + report(&calibration, &undecodable, &mut manifest); + manifest.calibration = Some(calibration.clone()); + session.write_manifest(&manifest)?; + println!("session: {}", dir.display()); + + if calibration.passed { + Ok(()) + } else { + Err(eyre::Report::msg(format!( + "sync self-test FAILED: the video disagrees with the clocks by up to {:.0} ms (tolerance {:.0} ms)", + calibration.max_abs_error_ms, calibration.tolerance_ms + ))) + } +} + +fn report(calibration: &Calibration, undecodable: &[f64], manifest: &mut Manifest) { + println!("video second | clock shown in video vs predicted"); + for sample in &calibration.samples { + println!( + " {:>6} | {:+.0} ms", + timecode::format(sample.video_s), + sample.error_ms + ); + } + + let mean = calibration + .samples + .iter() + .map(|sample| sample.error_ms) + .sum::() + / calibration.samples.len().max(1) as f64; + println!( + "{} samples, mean bias {mean:+.1} ms (the clock is painted slightly before it is shown), \ + worst {:.0} ms, tolerance {:.0} ms: {}", + calibration.samples.len(), + calibration.max_abs_error_ms, + calibration.tolerance_ms, + if calibration.passed { "PASS" } else { "FAIL" } + ); + + if !undecodable.is_empty() { + let note = format!( + "self-test could not read the clock at {} of {} sampled timestamps", + undecodable.len(), + undecodable.len() + calibration.samples.len() + ); + println!("note: {note}"); + manifest.warnings.push(note); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn luminance(ms: u64) -> Vec { + encode_cells(ms) + .iter() + .map(|white| if *white { 255 } else { 0 }) + .collect() + } + + #[test] + fn a_mac_window_and_display_share_units_and_a_windows_window_does_not() { + // a Mac: both in points, the video 2 pixels to the point + assert_eq!( + strip_units((100.0, 50.0), 2.0, 2.0, false), + ((100.0, 50.0), 2.0) + ); + + // Windows at 150 %: the display at 1920 px is 1280 points, and a point is 1.5 pixels + let ((x, y), unit) = strip_units((1920.0, 0.0), 1.0, 1.5, true); + assert_eq!((x, y, unit), (1280.0, 0.0, 1.5)); + + // so a window 80 points into that display puts its strip 120 pixels + 20 points in + let filter = strip_filter((1280.0 + 80.0, 80.0), (x, y), unit); + assert!(filter.starts_with("crop=1440:45:150:"), "{filter}"); + } + + #[test] + fn the_strip_roundtrips_any_clock_value() { + for ms in [ + 0, + 1, + 0xA5A5A5A5A5 & (WRAP_MS - 1), + 1_210_934_273, + WRAP_MS - 1, + ] { + assert_eq!(decode_cells(&luminance(ms)), Some(ms), "{ms}"); + } + } + + #[test] + fn decoding_tolerates_antialiasing_but_not_a_missed_crop() { + // slightly soft edges still decode + let mut soft = luminance(0b1011_0010); + for (index, level) in soft.iter_mut().enumerate() { + *level = if *level == 255 { + 230 + (index % 5) as u8 + } else { + 10 + (index % 7) as u8 + }; + } + assert_eq!(decode_cells(&soft), Some(0b1011_0010)); + + // flat grey is a crop that missed the strip + assert_eq!(decode_cells(&[128; CELLS]), None); + assert_eq!(decode_cells(&[0; 10]), None, "wrong length"); + } + + #[test] + fn wrap_around_differences_are_small() { + assert_eq!(wrapped_difference_ms(1_050, 1_000), 50); + assert_eq!(wrapped_difference_ms(950, 1_000), -50); + // the strip wrapped between the prediction and the observation + assert_eq!(wrapped_difference_ms(5, WRAP_MS - 5), 10); + assert_eq!(wrapped_difference_ms(WRAP_MS - 5, 5), -10); + } + + #[test] + fn the_crop_filter_targets_the_strip_in_display_pixels() { + // window at (80, 80) on a display at the origin, 1x + assert_eq!( + strip_filter((80.0, 80.0), (0.0, 0.0), 1.0), + "crop=960:30:100:205,scale=40:1:flags=area" + ); + // the same window on a display to the left, at 2x + assert_eq!( + strip_filter((-1720.0, 80.0), (-1800.0, 0.0), 2.0), + "crop=1920:60:200:410,scale=40:1:flags=area" + ); + } + + #[test] + fn evaluation_passes_within_tolerance_and_reports_the_worst_error() { + let readings = vec![ + Reading { + video_s: 1.0, + predicted_ms: 10_000, + observed_ms: 9_970, + }, + Reading { + video_s: 1.5, + predicted_ms: 10_500, + observed_ms: 10_460, + }, + ]; + let calibration = evaluate(&readings, 100.0); + assert!(calibration.passed); + assert!((calibration.max_abs_error_ms - 40.0).abs() < 1e-9); + assert_eq!(calibration.samples[0].error_ms, -30.0); + + assert!(!evaluate(&readings, 35.0).passed); + assert!(!evaluate(&[], 100.0).passed, "no readings is not a pass"); + } +}