diff --git a/Cargo.lock b/Cargo.lock index 5e02d921..87c905ce 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -109,6 +109,15 @@ version = "2.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c4512299f36f043ab09a583e57bceb5a5aab7a73db1805848e8fef3c9e8c78b3" +[[package]] +name = "block2" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdeb9d870516001442e364c5220d3574d2da8dc765554b4a617230d33fa58ef5" +dependencies = [ + "objc2", +] + [[package]] name = "bumpalo" version = "3.20.2" @@ -260,6 +269,18 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "dispatch2" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e0e367e4e7da84520dedcac1901e4da967309406d1e51017ae1abfb97adbd38" +dependencies = [ + "bitflags", + "block2", + "libc", + "objc2", +] + [[package]] name = "dyn-clone" version = "1.0.20" @@ -461,14 +482,21 @@ dependencies = [ name = "komorebi-recorder" version = "0.1.0" dependencies = [ + "block2", "chrono", "clap", "color-eyre", "crossbeam-channel", "dirs", + "dispatch2", "libc", + "objc2", + "objc2-av-foundation", "objc2-core-foundation", "objc2-core-graphics", + "objc2-core-media", + "objc2-foundation", + "objc2-screen-capture-kit", "parking_lot", "schemars", "serde", @@ -558,6 +586,51 @@ dependencies = [ "autocfg", ] +[[package]] +name = "objc2" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a12a8ed07aefc768292f076dc3ac8c48f3781c8f2d5851dd3d98950e8c5a89f" +dependencies = [ + "objc2-encode", +] + +[[package]] +name = "objc2-av-foundation" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "478ae33fcac9df0a18db8302387c666b8ef08a3e2d62b510ca4fc278a384b6c0" +dependencies = [ + "bitflags", + "block2", + "objc2", + "objc2-core-foundation", + "objc2-core-media", + "objc2-foundation", +] + +[[package]] +name = "objc2-core-audio" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1eebcea8b0dbff5f7c8504f3107c68fc061a3eb44932051c8cf8a68d969c3b2" +dependencies = [ + "dispatch2", + "objc2", + "objc2-core-audio-types", + "objc2-core-foundation", +] + +[[package]] +name = "objc2-core-audio-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a89f2ec274a0cf4a32642b2991e8b351a404d290da87bb6a9a9d8632490bd1c" +dependencies = [ + "bitflags", + "objc2", +] + [[package]] name = "objc2-core-foundation" version = "0.3.2" @@ -565,6 +638,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2a180dd8642fa45cdb7dd721cd4c11b1cadd4929ce112ebd8b9f5803cc79d536" dependencies = [ "bitflags", + "dispatch2", + "objc2", ] [[package]] @@ -574,8 +649,82 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e022c9d066895efa1345f8e33e584b9f958da2fd4cd116792e15e07e4720a807" dependencies = [ "bitflags", + "dispatch2", "libc", + "objc2", "objc2-core-foundation", + "objc2-io-surface", +] + +[[package]] +name = "objc2-core-media" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05ec576860167a15dd9fce7fbee7512beb4e31f532159d3482d1f9c6caedf31d" +dependencies = [ + "bitflags", + "dispatch2", + "objc2", + "objc2-core-audio", + "objc2-core-audio-types", + "objc2-core-foundation", + "objc2-core-video", +] + +[[package]] +name = "objc2-core-video" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d425caf1df73233f29fd8a5c3e5edbc30d2d4307870f802d18f00d83dc5141a6" +dependencies = [ + "bitflags", + "objc2", + "objc2-core-foundation", + "objc2-core-graphics", + "objc2-io-surface", +] + +[[package]] +name = "objc2-encode" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef25abbcd74fb2609453eb695bd2f860d389e457f67dc17cafc8b8cbc89d0c33" + +[[package]] +name = "objc2-foundation" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3e0adef53c21f888deb4fa59fc59f7eb17404926ee8a6f59f5df0fd7f9f3272" +dependencies = [ + "bitflags", + "objc2", +] + +[[package]] +name = "objc2-io-surface" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "180788110936d59bab6bd83b6060ffdfffb3b922ba1396b312ae795e1de9d81d" +dependencies = [ + "bitflags", + "objc2", + "objc2-core-foundation", +] + +[[package]] +name = "objc2-screen-capture-kit" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74b7c5390f477482f001bc354d6571a70db7e4f8d5288e860c45521fbce11394" +dependencies = [ + "block2", + "dispatch2", + "libc", + "objc2", + "objc2-core-foundation", + "objc2-core-graphics", + "objc2-core-media", + "objc2-foundation", ] [[package]] diff --git a/dependencies.json b/dependencies.json index bd3be8f8..192f8b6c 100644 --- a/dependencies.json +++ b/dependencies.json @@ -33,6 +33,7 @@ "crossbeam-utils 0.8.21 registry+https://github.com/rust-lang/crates.io-index", "dirs 6.0.0 registry+https://github.com/rust-lang/crates.io-index", "dirs-sys 0.5.0 registry+https://github.com/rust-lang/crates.io-index", + "dispatch2 0.3.1 registry+https://github.com/rust-lang/crates.io-index", "dyn-clone 1.0.20 registry+https://github.com/rust-lang/crates.io-index", "errno 0.3.14 registry+https://github.com/rust-lang/crates.io-index", "eyre 0.6.12 registry+https://github.com/rust-lang/crates.io-index", @@ -49,8 +50,12 @@ "lock_api 0.4.14 registry+https://github.com/rust-lang/crates.io-index", "miniz_oxide 0.8.9 registry+https://github.com/rust-lang/crates.io-index", "num-traits 0.2.19 registry+https://github.com/rust-lang/crates.io-index", + "objc2-av-foundation 0.3.2 registry+https://github.com/rust-lang/crates.io-index", "objc2-core-foundation 0.3.2 registry+https://github.com/rust-lang/crates.io-index", "objc2-core-graphics 0.3.2 registry+https://github.com/rust-lang/crates.io-index", + "objc2-core-media 0.3.2 registry+https://github.com/rust-lang/crates.io-index", + "objc2-core-video 0.3.2 registry+https://github.com/rust-lang/crates.io-index", + "objc2-screen-capture-kit 0.3.2 registry+https://github.com/rust-lang/crates.io-index", "object 0.37.3 registry+https://github.com/rust-lang/crates.io-index", "once_cell 1.21.4 registry+https://github.com/rust-lang/crates.io-index", "once_cell_polyfill 1.70.2 registry+https://github.com/rust-lang/crates.io-index", @@ -129,6 +134,7 @@ "autocfg 1.5.0 registry+https://github.com/rust-lang/crates.io-index", "backtrace 0.3.76 registry+https://github.com/rust-lang/crates.io-index", "bitflags 2.11.1 registry+https://github.com/rust-lang/crates.io-index", + "block2 0.6.2 registry+https://github.com/rust-lang/crates.io-index", "cfg-if 1.0.4 registry+https://github.com/rust-lang/crates.io-index", "chrono 0.4.44 registry+https://github.com/rust-lang/crates.io-index", "clap 4.6.1 registry+https://github.com/rust-lang/crates.io-index", @@ -143,6 +149,7 @@ "crossbeam-utils 0.8.21 registry+https://github.com/rust-lang/crates.io-index", "dirs 6.0.0 registry+https://github.com/rust-lang/crates.io-index", "dirs-sys 0.5.0 registry+https://github.com/rust-lang/crates.io-index", + "dispatch2 0.3.1 registry+https://github.com/rust-lang/crates.io-index", "dyn-clone 1.0.20 registry+https://github.com/rust-lang/crates.io-index", "errno 0.3.14 registry+https://github.com/rust-lang/crates.io-index", "eyre 0.6.12 registry+https://github.com/rust-lang/crates.io-index", @@ -161,8 +168,15 @@ "memoffset 0.9.1 registry+https://github.com/rust-lang/crates.io-index", "miniz_oxide 0.8.9 registry+https://github.com/rust-lang/crates.io-index", "num-traits 0.2.19 registry+https://github.com/rust-lang/crates.io-index", + "objc2 0.6.4 registry+https://github.com/rust-lang/crates.io-index", + "objc2-av-foundation 0.3.2 registry+https://github.com/rust-lang/crates.io-index", "objc2-core-foundation 0.3.2 registry+https://github.com/rust-lang/crates.io-index", "objc2-core-graphics 0.3.2 registry+https://github.com/rust-lang/crates.io-index", + "objc2-core-media 0.3.2 registry+https://github.com/rust-lang/crates.io-index", + "objc2-core-video 0.3.2 registry+https://github.com/rust-lang/crates.io-index", + "objc2-encode 4.1.0 registry+https://github.com/rust-lang/crates.io-index", + "objc2-foundation 0.3.2 registry+https://github.com/rust-lang/crates.io-index", + "objc2-screen-capture-kit 0.3.2 registry+https://github.com/rust-lang/crates.io-index", "object 0.37.3 registry+https://github.com/rust-lang/crates.io-index", "once_cell 1.21.4 registry+https://github.com/rust-lang/crates.io-index", "once_cell_polyfill 1.70.2 registry+https://github.com/rust-lang/crates.io-index", @@ -241,9 +255,14 @@ [ "Zlib", [ + "dispatch2 0.3.1 registry+https://github.com/rust-lang/crates.io-index", "miniz_oxide 0.8.9 registry+https://github.com/rust-lang/crates.io-index", + "objc2-av-foundation 0.3.2 registry+https://github.com/rust-lang/crates.io-index", "objc2-core-foundation 0.3.2 registry+https://github.com/rust-lang/crates.io-index", - "objc2-core-graphics 0.3.2 registry+https://github.com/rust-lang/crates.io-index" + "objc2-core-graphics 0.3.2 registry+https://github.com/rust-lang/crates.io-index", + "objc2-core-media 0.3.2 registry+https://github.com/rust-lang/crates.io-index", + "objc2-core-video 0.3.2 registry+https://github.com/rust-lang/crates.io-index", + "objc2-screen-capture-kit 0.3.2 registry+https://github.com/rust-lang/crates.io-index" ] ] ] diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index b132559a..1dda543d 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -75,6 +75,14 @@ The files beside it are the raw records, and they are written as the data arrive (`komorebi.log`, the exact byte range recorded), its events (`events.jsonl`, with full `State` snapshots in `states/` only when they changed), and what happened to the recording itself (`recorder.jsonl`). The OS trace is under `oslog/`, one file per stream. +Each display's video is `video-[-].mp4` with a `frames-[-].csv` beside it holding every frame's timestamp, +one segment to a file: a display that is unplugged and returns starts a new segment instead of extending the old one. + +## Recording + +The recorder starts its sources in an order that makes the sources that are not video a superset of the video's span +(the clock anchor, the OS trace streams, the subscription to komorebi and its first `State`, komorebi's log offset, and +last the video), and stops them in the reverse order. ## Rules and the layer that enforces them diff --git a/docs/PLATFORM.md b/docs/PLATFORM.md index 6d822283..e6a16bf4 100644 --- a/docs/PLATFORM.md +++ b/docs/PLATFORM.md @@ -46,6 +46,21 @@ Video frame times and the OS trace are on the master clock already on both hosts `machTimestamp` on a Mac; Windows.Graphics.Capture on Windows). The field names ending in `_mach_ns` are spelled that way on every host: see [the architecture](ARCHITECTURE.md#the-master-clock). +### Capturing the screen + +| | macOS | Windows | +|---|---|---| +| API | ScreenCaptureKit, through `objc2` bindings, and AVFoundation's `AVAssetWriter` | Windows.Graphics.Capture (Windows 10 version 1803 or later), and a Media Foundation MP4 sink | +| permission | Screen Recording, granted to the terminal or app that launches the binary | none | +| default codec | HEVC | H.264 | +| other codec | H.264 (`--codec h264`) | HEVC is refused with an error: Media Foundation's MP4 sink does not write it | +| what is captured | each display, excluding the recorder's own processes by name | each monitor whole, so the program's windows ask to be left out (see below) | +| a still screen | a healthy stream still calls back about 30 times a second, so silence for four seconds means a stalled stream | Windows sends nothing while the picture does not change, so silence says nothing there | + +The video of a display is written as fragmented MP4 on both hosts, in segments, so that the part written before a crash +can still be played. Because ScreenCaptureKit only emits frames when the screen changes, the macOS capture re-submits the +last frame so that a static display still flushes fragments. + ### The OS trace | | macOS | Windows | diff --git a/komorebi-recorder/Cargo.toml b/komorebi-recorder/Cargo.toml index a2698bbf..2b8c6bbc 100644 --- a/komorebi-recorder/Cargo.toml +++ b/komorebi-recorder/Cargo.toml @@ -17,7 +17,24 @@ serde = { workspace = true } serde_json = { workspace = true } [target.'cfg(target_os = "macos")'.dependencies] +block2 = "0.6" +dispatch2 = "0.3" libc = "0.2" +objc2 = { version = "0.6", default-features = false, features = ["std"] } +objc2-av-foundation = { version = "0.3", default-features = false, features = [ + "std", + "block2", + "objc2-core-media", + "AVAsset", + "AVAssetReader", + "AVAssetReaderOutput", + "AVAssetTrack", + "AVAssetWriter", + "AVAssetWriterInput", + "objc2-core-foundation", + "AVMediaFormat", + "AVVideoSettings", +] } objc2-core-foundation = { version = "0.3", default-features = false, features = [ "std", "CFArray", @@ -37,6 +54,37 @@ objc2-core-graphics = { version = "0.3", default-features = false, features = [ "CGSession", "CGWindow", ] } +objc2-core-media = { version = "0.3", default-features = false, features = [ + "std", + "objc2-core-video", + "CMBase", + "CMSampleBuffer", + "CMSync", + "CMTime", + "CMTimeRange", +] } +objc2-foundation = { version = "0.3", default-features = false, features = [ + "std", + "NSArray", + "NSDictionary", + "NSEnumerator", + "NSError", + "NSObject", + "NSString", + "NSURL", + "NSValue", +] } +objc2-screen-capture-kit = { version = "0.3", default-features = false, features = [ + "std", + "block2", + "dispatch2", + "libc", + "objc2-core-foundation", + "objc2-core-graphics", + "objc2-core-media", + "SCShareableContent", + "SCStream", +] } [target.'cfg(windows)'.dependencies] uds_windows = "1" diff --git a/komorebi-recorder/src/capture/manager.rs b/komorebi-recorder/src/capture/manager.rs new file mode 100644 index 00000000..32794ee6 --- /dev/null +++ b/komorebi-recorder/src/capture/manager.rs @@ -0,0 +1,1506 @@ +//! Keeps every wanted display recording while displays come and go. +//! +//! A laptop that is docked and undocked, woken with the lid opened, or locked and unlocked +//! changes its set of displays and kills `ScreenCaptureKit` streams at unpredictable moments. The +//! rule here is that *nothing already captured is ever put at risk by that*: +//! +//! - every recording is a **segment**: one display, one file, one contiguous span. When a display +//! disappears or its stream dies, that segment is finalized immediately (so it is complete and +//! playable no matter what happens next) and the other displays keep recording untouched; +//! - when a display (re)appears it starts a **new segment** in a new file, after a short settle +//! delay so a display that is still waking up is not started half-formed; +//! - a start that fails (the display is not ready, the screen is locked) is retried with backoff +//! and reported once, not every attempt; +//! - an unreadable display list changes nothing, since "I could not ask" is not "they all left". +//! +//! Time is passed in rather than read, so the whole policy is testable with a scripted backend. +//! +//! # What ends a capture +//! +//! [`DisplayManager::poll`] looks at every running capture and applies the first rule that +//! matches, in this order: +//! +//! | situation | [`EndReason`] | incident kind ([`kind`]) | what happens next | +//! |---|---|---|---| +//! | the capture reported an error | `StreamError`, ended when the error was reported | `capture-stopped` | a new segment after a one second backoff, if the display is still present and awake | +//! | the display is missing from the display list | `DisplayDisconnected` | `display-disconnected` | a new segment when the display returns, after the settle delay | +//! | the display is asleep | `DisplayAsleep` | `display-asleep` | a new segment when it wakes, after the settle delay | +//! | no sign of life for longer than four seconds | `StreamError`, ended when it last showed life | `capture-stopped` | as for an error | +//! | the display's size changed | `Reconfigured` | `display-reconfigured` | a new segment in the same poll | +//! +//! The order matters when several apply at once: a display that vanished is a disconnect even if +//! its stream had also gone quiet. The stall rule applies only to a capture that reports a sign of +//! life ([`RunningCapture::last_signal_mach_ns`]); the macOS backend does, and the Windows backend +//! does not, so a Windows capture is never replaced for silence. +//! +//! [`DisplayManager::restart_all`] is the one way a capture ends that is not found by looking: the +//! caller knows the machine slept and asks for every capture to be replaced. +//! +//! # Timing +//! +//! | what | how long | +//! |---|---| +//! | settle: a display that appeared after recording began must be present this long before it is started, so that one still waking up is not started half-formed | one second | +//! | stall: a capture with no sign of life, not even an idle callback, for this long has stalled | four seconds | +//! | retry backoff after a failed start or a dead stream | one second, doubling up to ten seconds | +//! +//! The first display list of a recording is not delayed ([`DisplayManager::start`]). +//! +//! # What this does not do +//! +//! A display's consecutive segments are not guaranteed to be gap-free: after a disconnect, a sleep +//! or a dead stream, the settle delay and the backoff are time in which that display is not +//! recorded (a resolution change is the exception: the next segment starts in the same poll). +//! What is guaranteed is that what was captured before is complete. The manager does not poll by +//! itself: the caller calls [`DisplayManager::poll`] several times a second and records the +//! returned [`Draft`]s. + +use crate::capture::CaptureBackend; +use crate::capture::CaptureConfig; +use crate::capture::DisplayDescriptor; +use crate::capture::DisplayResult; +use crate::capture::EndReason; +use crate::capture::RunningCapture; +use crate::incidents::Draft; +use crate::incidents::kind; +use color_eyre::eyre; +use std::collections::BTreeMap; +use std::collections::BTreeSet; +use std::path::PathBuf; + +/// How long a display that appeared after recording began must be present before it is started. +const SETTLE_NS: u64 = 1_000_000_000; +/// A capture that shows no sign of life (not even an idle callback, which a healthy stream makes +/// about thirty times a second even on a static screen) for this long has stalled. +const STALL_NS: u64 = 4_000_000_000; +/// The first retry after a stream dies or a start fails, doubling up to [`MAX_BACKOFF_NS`]. +const FIRST_BACKOFF_NS: u64 = 1_000_000_000; +const MAX_BACKOFF_NS: u64 = 10_000_000_000; + +/// Which displays to record. +/// +/// `record` follows [`Follow::All`] unless it is given display ids, and then [`Follow::Only`] +/// those. +/// +/// # Example +/// +/// ```rust +/// use komorebi_recorder::capture::manager::Follow; +/// use std::collections::BTreeSet; +/// +/// assert!(Follow::All.wants(7)); +/// +/// let only = Follow::Only(BTreeSet::from([2])); +/// assert!(only.wants(2)); +/// assert!(!only.wants(7)); +/// ``` +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Follow { + /// Every display, including ones connected while recording. + All, + /// Only these displays (and only when they are present). A selected display that is + /// unplugged and plugged back in is recorded again; one that was not selected never is. + Only(BTreeSet), +} + +impl Follow { + /// Returns `true` if and only if the display with this id is to be recorded. + #[must_use] + pub fn wants(&self, display: u32) -> bool { + match self { + Follow::All => true, + Follow::Only(set) => set.contains(&display), + } + } +} + +struct Capturing { + capture: Box, + /// The display as the probe described it when the segment began. + probed: DisplayDescriptor, +} + +struct Pending { + next_attempt_ns: u64, + failures: u32, + /// Whether the current outage has been reported yet. + reported: bool, +} + +/// Keeps every wanted display recording as displays come and go, one [`DisplayResult`] per segment. +/// +/// Generic over the [`CaptureBackend`] so that the whole policy runs against a scripted backend. +/// The caller drives it: [`start`](DisplayManager::start) once, then +/// [`poll`](DisplayManager::poll) several times a second, +/// [`restart_all`](DisplayManager::restart_all) after a sleep, +/// [`checkpoint`](DisplayManager::checkpoint) now and then for the session manifest, and +/// [`shutdown`](DisplayManager::shutdown) at the end. Every method that can notice something +/// returns [`Draft`]s, incidents not yet timestamped, for the caller to log; the manager itself +/// writes nothing but the captures' files. See the [module documentation](self) for the rules. +/// +/// Every `now_ns` is a time on the master clock (see [`crate::clock`]), passed in so that the +/// manager never reads a clock itself. +/// +/// # Example +/// +/// A laptop with two displays, one of which is unplugged and plugged back in. The backend is a +/// script: the example changes the list of attached displays between polls. +/// +/// ```rust +/// use color_eyre::eyre; +/// use komorebi_recorder::capture::manager::{DisplayManager, Follow}; +/// use komorebi_recorder::capture::{ +/// CaptureBackend, CaptureConfig, DisplayDescriptor, EndReason, RunningCapture, +/// }; +/// use komorebi_recorder::incidents::kind; +/// use std::cell::RefCell; +/// use std::path::{Path, PathBuf}; +/// use std::rc::Rc; +/// # use komorebi_recorder::capture::{ +/// # CaptureStats, Codec, DisplayResult, LatencyStats, frames_file_name, video_file_name, +/// # }; +/// # use std::sync::Arc; +/// # +/// # // A capture that writes nothing. Its first frame is at a time that makes the results +/// # // sort by segment and then by display. +/// # struct Recording { +/// # display: u32, +/// # segment: u32, +/// # } +/// # +/// # impl Recording { +/// # fn result(&self, reason: Option) -> DisplayResult { +/// # DisplayResult { +/// # display: DisplayDescriptor { +/// # id: self.display, +/// # frame: [0.0, 0.0, 100.0, 100.0], +/// # width_points: 100, +/// # height_points: 100, +/// # asleep: false, +/// # scale: 1.0, +/// # }, +/// # segment: self.segment, +/// # scale: 1.0, +/// # width_px: 100, +/// # height_px: 100, +/// # codec: Codec::H264, +/// # file: video_file_name(self.display, self.segment), +/// # frames_file: frames_file_name(self.display, self.segment), +/// # video_t0_mach_ns: Some(u64::from(self.segment) * 10 + u64::from(self.display)), +/// # video_end_mach_ns: None, +/// # end_reason: reason, +/// # frames: 0, +/// # dropped: 0, +/// # idle_callbacks: 0, +/// # latency: LatencyStats::default(), +/// # error: None, +/// # } +/// # } +/// # } +/// # +/// # impl RunningCapture for Recording { +/// # fn display_id(&self) -> u32 { +/// # self.display +/// # } +/// # fn segment(&self) -> u32 { +/// # self.segment +/// # } +/// # fn stats(&self) -> Arc { +/// # Arc::default() +/// # } +/// # fn partial(&self) -> DisplayResult { +/// # self.result(None) +/// # } +/// # fn stop(self: Box, reason: EndReason, _end: Option) -> DisplayResult { +/// # self.result(Some(reason)) +/// # } +/// # } +/// +/// const SECOND: u64 = 1_000_000_000; +/// +/// fn display(id: u32) -> DisplayDescriptor { +/// DisplayDescriptor { +/// id, +/// frame: [0.0, 0.0, 1920.0, 1080.0], +/// width_points: 1920, +/// height_points: 1080, +/// asleep: false, +/// scale: 1.0, +/// } +/// } +/// +/// /// The displays that are attached, shared with the example so that it can unplug them. +/// struct Screen(Rc>>); +/// +/// impl CaptureBackend for Screen { +/// fn active_displays(&self) -> eyre::Result> { +/// Ok(self.0.borrow().clone()) +/// } +/// +/// fn start( +/// &self, +/// display: &DisplayDescriptor, +/// segment: u32, +/// _config: &CaptureConfig, +/// _dir: &Path, +/// ) -> eyre::Result> { +/// Ok(Box::new(Recording { display: display.id, segment })) +/// } +/// } +/// +/// let attached = Rc::new(RefCell::new(vec![display(1), display(2)])); +/// let mut manager = DisplayManager::new( +/// Screen(attached.clone()), +/// CaptureConfig::default(), +/// PathBuf::from("/session"), +/// Follow::All, +/// ); +/// +/// // Both displays are recorded at once, with no settle delay. +/// let drafts = manager.start(0)?; +/// assert_eq!(drafts.len(), 2); +/// assert!(drafts.iter().all(|draft| draft.kind == kind::CAPTURE_STARTED)); +/// assert_eq!(manager.recording_count(), 2); +/// +/// // Display 2 is unplugged. Its segment is finalized on the spot, and display 1 carries on. +/// attached.borrow_mut().retain(|display| display.id != 2); +/// let drafts = manager.poll(5 * SECOND); +/// assert_eq!(drafts.len(), 1); +/// assert_eq!(drafts[0].kind, kind::DISPLAY_DISCONNECTED); +/// assert_eq!(drafts[0].display, Some(2)); +/// assert_eq!(manager.recording_count(), 1); +/// +/// // It is plugged back in. It is reported as available again, not as new, and is recorded as a +/// // new segment once it has been present for a second. +/// attached.borrow_mut().push(display(2)); +/// let drafts = manager.poll(20 * SECOND); +/// assert_eq!(drafts[0].kind, kind::DISPLAY_AWAKE); +/// assert_eq!(manager.recording_count(), 1, "still settling"); +/// let drafts = manager.poll(21 * SECOND); +/// assert_eq!(drafts[0].kind, kind::CAPTURE_STARTED); +/// assert_eq!(manager.recording_count(), 2); +/// +/// // Stopping returns every segment in time order. Display 2 has two, in two files. +/// let results = manager.shutdown(); +/// let segments: Vec<_> = results +/// .iter() +/// .map(|result| (result.file.as_str(), result.end_reason.expect("every segment ended"))) +/// .collect(); +/// assert_eq!( +/// segments, +/// [ +/// ("video-1.mp4", EndReason::Stopped), +/// ("video-2.mp4", EndReason::DisplayDisconnected), +/// ("video-2-2.mp4", EndReason::Stopped), +/// ] +/// ); +/// # Ok::<(), Box>(()) +/// ``` +pub struct DisplayManager { + backend: B, + config: CaptureConfig, + dir: PathBuf, + follow: Follow, + active: BTreeMap, + pending: BTreeMap, + next_segment: BTreeMap, + finished: Vec, + /// Every display id ever seen, so one that returns is reported as returning, not as new. + seen: BTreeSet, + probe_failed: bool, +} + +fn backoff_ns(failures: u32) -> u64 { + FIRST_BACKOFF_NS + .saturating_mul(1 << failures.saturating_sub(1).min(8)) + .min(MAX_BACKOFF_NS) +} + +impl DisplayManager { + /// Creates a manager that records the displays `follow` selects through `backend`, with + /// `config`, into `dir`. + /// + /// `dir` is the session directory the segment files are written to. Nothing is started and the + /// backend is not asked anything: see [`start`](DisplayManager::start). + pub fn new(backend: B, config: CaptureConfig, dir: PathBuf, follow: Follow) -> Self { + Self { + backend, + config, + dir, + follow, + active: BTreeMap::new(), + pending: BTreeMap::new(), + next_segment: BTreeMap::new(), + finished: Vec::new(), + seen: BTreeSet::new(), + probe_failed: false, + } + } + + /// Returns the displays that are present right now and wanted: followed (see [`Follow`]) and + /// not asleep. + /// + /// # Errors + /// + /// Returns the backend's error when it cannot list the displays + /// ([`CaptureBackend::active_displays`]). + pub fn wanted_displays(&self) -> eyre::Result> { + Ok(self + .backend + .active_displays()? + .into_iter() + .filter(|display| self.follow.wants(display.id) && !display.asleep) + .collect()) + } + + /// Begins recording every wanted display that is present, with no settle delay, and returns the + /// incidents that came of it. + /// + /// Call it once, before the first [`poll`](DisplayManager::poll). A display whose capture fails + /// to start is not an error here: the failure comes back as a `capture-failed` draft (once) and + /// the start is retried with a backoff by `poll`. A caller that needs at least one display + /// recording checks [`recording_count`](DisplayManager::recording_count) afterwards. + /// + /// # Errors + /// + /// Returns the backend's error when it cannot list the displays + /// ([`CaptureBackend::active_displays`]). Nothing has been started in that case. + pub fn start(&mut self, now_ns: u64) -> eyre::Result> { + for display in self.wanted_displays()? { + self.seen.insert(display.id); + self.pending.insert( + display.id, + Pending { + next_attempt_ns: now_ns, + failures: 0, + reported: false, + }, + ); + } + + Ok(self.poll(now_ns)) + } + + /// Reconciles what is recording with what is present, and returns the incidents that came of + /// it. Cheap when nothing changed; call it several times a second. + /// + /// Every poll first gives each running capture its + /// [`keepalive`](RunningCapture::keepalive). It then lists the displays. If listing fails, + /// nothing else changes and the failure is reported once as a `recorder` draft until a listing + /// succeeds again. Otherwise it ends the captures that have to end (see the [module + /// documentation](self) for the rules and their order) and starts the ones that are missing: + /// + /// - a display seen for the first time is reported as `display-connected`, and one that had + /// been seen before as `display-awake` ("available again"). Either is started after the + /// settle delay; + /// - a start that fails is reported once as `capture-failed`, whatever the number of + /// attempts, and retried after a backoff that doubles from one second up to ten; + /// - a start that succeeds is reported as `capture-started`, or as `capture-recovered` if it + /// follows a failure or a dead stream; + /// - a display that is asleep or not followed is left alone. + /// + /// A draft dated by [`Draft::at`] carries the time the thing happened when that is earlier + /// than the poll (a stream that died). + pub fn poll(&mut self, now_ns: u64) -> Vec { + let mut drafts = Vec::new(); + + for capturing in self.active.values() { + capturing.capture.keepalive(now_ns); + } + + let probe = match self.backend.active_displays() { + Ok(displays) => { + self.probe_failed = false; + displays + } + Err(error) => { + // not being able to ask is not the same as every display having left + if !self.probe_failed { + self.probe_failed = true; + drafts.push(Draft::new( + kind::RECORDER, + format!("could not list displays, keeping current captures: {error:#}"), + )); + } + return drafts; + } + }; + let present: BTreeMap = probe + .into_iter() + .map(|display| (display.id, display)) + .collect(); + + self.end_dead_captures(&present, now_ns, &mut drafts); + self.start_missing_captures(&present, now_ns, &mut drafts); + self.pending.retain(|id, _| present.contains_key(id)); + + drafts + } + + /// Finalize captures whose stream died, whose display left, or whose display changed size. + fn end_dead_captures( + &mut self, + present: &BTreeMap, + now_ns: u64, + drafts: &mut Vec, + ) { + let ids: Vec = self.active.keys().copied().collect(); + for id in ids { + let Some(capturing) = self.active.get(&id) else { + continue; + }; + + let stats = capturing.capture.stats(); + let stalled_for = capturing + .capture + .last_signal_mach_ns() + .map(|last| (last, now_ns.saturating_sub(last))) + .filter(|(_, silent_for)| *silent_for > STALL_NS); + + let (reason, end, detail) = if let Some(error) = stats.error() { + ( + EndReason::StreamError, + // end at the moment it died, not whenever we noticed + Some(stats.error_at_mach_ns().unwrap_or(now_ns)), + format!("display {id}: {error}"), + ) + } else if !present.contains_key(&id) { + ( + EndReason::DisplayDisconnected, + None, + format!("display {id} disconnected"), + ) + } else if present.get(&id).is_some_and(|display| display.asleep) { + ( + EndReason::DisplayAsleep, + None, + format!("display {id} went to sleep"), + ) + } else if let Some((last_signal, silent_for)) = stalled_for { + ( + EndReason::StreamError, + Some(last_signal), + format!( + "display {id}: the stream stalled (no frames or idle callbacks for {:.1}s)", + silent_for as f64 / 1e9 + ), + ) + } else if let Some(now_display) = present.get(&id) { + if capturing.probed.size_differs(now_display) { + ( + EndReason::Reconfigured, + None, + format!( + "display {id} changed from {}x{} to {}x{} points", + capturing.probed.width_points, + capturing.probed.height_points, + now_display.width_points, + now_display.height_points + ), + ) + } else { + continue; + } + } else { + continue; + }; + + let result = self.finalize(id, reason, end); + let event_kind = match reason { + EndReason::DisplayDisconnected => kind::DISPLAY_DISCONNECTED, + EndReason::DisplayAsleep => kind::DISPLAY_ASLEEP, + EndReason::Reconfigured => kind::DISPLAY_RECONFIGURED, + _ => kind::CAPTURE_STOPPED, + }; + let mut draft = Draft::new( + event_kind, + format!( + "{detail}; segment {} ({}) finalized with {} frames", + result.segment, result.file, result.frames + ), + ) + .display(id); + if let Some(end) = end { + draft = draft.at(end); + } + drafts.push(draft); + + // a display that is still here but whose stream died is retried; one that left, or + // went to sleep, is picked up again if and when it comes back + if present.get(&id).is_some_and(|display| !display.asleep) { + let delay = if reason == EndReason::Reconfigured { + 0 + } else { + FIRST_BACKOFF_NS + }; + self.pending.insert( + id, + Pending { + next_attempt_ns: now_ns + delay, + failures: u32::from(reason == EndReason::StreamError), + reported: reason != EndReason::Reconfigured, + }, + ); + } + } + } + + fn start_missing_captures( + &mut self, + present: &BTreeMap, + now_ns: u64, + drafts: &mut Vec, + ) { + for (id, display) in present { + if self.active.contains_key(id) || !self.follow.wants(*id) { + continue; + } + + // nothing to record until it wakes + if display.asleep { + continue; + } + + let returning = !self.seen.insert(*id); + let pending = self.pending.entry(*id).or_insert_with(|| { + drafts.push( + Draft::new( + if returning { + kind::DISPLAY_AWAKE + } else { + kind::DISPLAY_CONNECTED + }, + format!( + "display {id} {} ({}x{} points at {}, {})", + if returning { + "is available again" + } else { + "connected" + }, + display.width_points, + display.height_points, + display.frame[0], + display.frame[1] + ), + ) + .display(*id), + ); + // a display that just appeared may still be waking up: let it settle first + Pending { + next_attempt_ns: now_ns + SETTLE_NS, + failures: 0, + reported: false, + } + }); + + if now_ns < pending.next_attempt_ns { + continue; + } + + let segment = self.next_segment.get(id).copied().unwrap_or(1); + match self + .backend + .start(display, segment, &self.config, &self.dir) + { + Ok(capture) => { + let recovered = pending.reported; + self.next_segment.insert(*id, segment + 1); + self.pending.remove(id); + self.active.insert( + *id, + Capturing { + capture, + probed: display.clone(), + }, + ); + + drafts.push( + Draft::new( + if recovered { + kind::CAPTURE_RECOVERED + } else { + kind::CAPTURE_STARTED + }, + format!( + "display {id}: recording segment {segment} to {}", + crate::capture::video_file_name(*id, segment) + ), + ) + .display(*id), + ); + } + Err(error) => { + pending.failures += 1; + pending.next_attempt_ns = now_ns + backoff_ns(pending.failures); + + if !pending.reported { + pending.reported = true; + drafts.push( + Draft::new( + kind::CAPTURE_FAILED, + format!("display {id}: could not start recording, will keep retrying: {error:#}"), + ) + .display(*id), + ); + } + } + } + } + } + + fn finalize(&mut self, id: u32, reason: EndReason, end_mach_ns: Option) -> DisplayResult { + let Some(capturing) = self.active.remove(&id) else { + unreachable!("only called for ids taken from the active set"); + }; + + let result = capturing.capture.stop(reason, end_mach_ns); + self.finished.push(result.clone()); + result + } + + /// Ends every capture and starts fresh ones shortly after, for when the machine has been + /// asleep: streams are not trusted across a sleep, because they can stall silently. + /// + /// Each running capture is stopped with `reason` (the recorder passes + /// [`EndReason::Resumed`]) and its display is started again after the settle delay, as a new + /// segment. A display that did not come back is not resurrected: the next + /// [`poll`](DisplayManager::poll) drops the pending start of a display that is no longer + /// present. A start after a restart is reported as `capture-started`. + pub fn restart_all(&mut self, now_ns: u64, reason: EndReason) -> Vec { + let mut drafts = Vec::new(); + + for id in self.active.keys().copied().collect::>() { + let result = self.finalize(id, reason, None); + drafts.push( + Draft::new( + kind::CAPTURE_STOPPED, + format!( + "display {id}: {reason}; segment {} ({}) finalized with {} frames and a new one will start", + result.segment, result.file, result.frames + ), + ) + .display(id), + ); + + self.pending.insert( + id, + Pending { + next_attempt_ns: now_ns + SETTLE_NS, + failures: 0, + reported: false, + }, + ); + } + + drafts + } + + /// Returns the number of displays being recorded right now. + #[must_use] + pub fn recording_count(&self) -> usize { + self.active.len() + } + + /// Returns the ids of the displays waiting for another attempt after a failed start or a dead + /// stream. + #[must_use] + pub fn retrying(&self) -> Vec { + self.pending + .iter() + .filter(|(_, pending)| pending.reported) + .map(|(id, _)| *id) + .collect() + } + + /// Returns `(frames, dropped)` summed over every segment so far, finished or live. + #[must_use] + pub fn totals(&self) -> (u64, u64) { + let (mut frames, mut dropped) = (0, 0); + for result in &self.finished { + frames += result.frames; + dropped += result.dropped; + } + + for capturing in self.active.values() { + let stats = capturing.capture.stats(); + frames += stats.frames.load(std::sync::atomic::Ordering::Relaxed); + dropped += stats.dropped.load(std::sync::atomic::Ordering::Relaxed); + } + + (frames, dropped) + } + + /// Returns every segment so far, with live ones in their current state + /// ([`RunningCapture::partial`]). + /// + /// The segments are in time order: by the time of their first frame (those with none last), + /// then display id, then segment number. For manifest checkpoints: it does not end anything. + #[must_use] + pub fn checkpoint(&self) -> Vec { + let mut results = self.finished.clone(); + results.extend( + self.active + .values() + .map(|capturing| capturing.capture.partial()), + ); + sort_segments(&mut results); + results + } + + /// Stops everything and returns every segment in time order, as [`checkpoint`] orders them. + /// + /// Each running capture is stopped with [`EndReason::Stopped`]. Afterwards no capture is + /// running, no start is pending and the finished segments have been handed over, so a second + /// call returns an empty list. + /// + /// [`checkpoint`]: DisplayManager::checkpoint + pub fn shutdown(&mut self) -> Vec { + let ids: Vec = self.active.keys().copied().collect(); + for id in ids { + self.finalize(id, EndReason::Stopped, None); + } + + self.pending.clear(); + let mut results = std::mem::take(&mut self.finished); + sort_segments(&mut results); + results + } +} + +fn sort_segments(results: &mut [DisplayResult]) { + results.sort_by_key(|result| { + ( + result.video_t0_mach_ns.unwrap_or(u64::MAX), + result.display.id, + result.segment, + ) + }); +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::capture::CaptureStats; + use crate::capture::LatencyStats; + use crate::capture::frames_file_name; + use crate::capture::video_file_name; + use std::cell::Cell; + use std::cell::RefCell; + use std::path::Path; + use std::rc::Rc; + use std::sync::Arc; + + const SEC: u64 = 1_000_000_000; + + fn display(id: u32, width: i64, height: i64, x: f64) -> DisplayDescriptor { + DisplayDescriptor { + id, + frame: [x, 0.0, width as f64, height as f64], + width_points: width, + height_points: height, + asleep: false, + scale: 1.0, + } + } + + fn asleep(mut display: DisplayDescriptor) -> DisplayDescriptor { + display.asleep = true; + display + } + + fn builtin() -> DisplayDescriptor { + display(1, 1800, 1169, -1800.0) + } + + fn external() -> DisplayDescriptor { + display(2, 3840, 1620, 0.0) + } + + /// (display, segment, reason, end) of a stopped capture + type Stop = (u32, u32, EndReason, Option); + + #[derive(Default)] + struct World { + displays: RefCell>, + /// displays whose start currently fails + failing: RefCell>, + probe_fails: Cell, + /// (display, segment) for every start + starts: RefCell>, + /// (display, segment, reason, end) for every stop + stops: RefCell>, + /// the stats of every live capture, by display, so a test can kill one + stats: RefCell>>, + /// when each live capture last showed a sign of life; absent means the mock cannot tell + signals: RefCell>, + /// the display of every keepalive call + keepalives: RefCell>, + } + + struct MockBackend(Rc); + + struct MockCapture { + id: u32, + segment: u32, + stats: Arc, + world: Rc, + } + + impl RunningCapture for MockCapture { + fn display_id(&self) -> u32 { + self.id + } + + fn segment(&self) -> u32 { + self.segment + } + + fn stats(&self) -> Arc { + self.stats.clone() + } + + fn partial(&self) -> DisplayResult { + self.result(None, None) + } + + fn last_signal_mach_ns(&self) -> Option { + self.world.signals.borrow().get(&self.id).copied() + } + + fn keepalive(&self, _now_ns: u64) { + self.world.keepalives.borrow_mut().push(self.id); + } + + fn stop(self: Box, reason: EndReason, end_mach_ns: Option) -> DisplayResult { + self.world + .stops + .borrow_mut() + .push((self.id, self.segment, reason, end_mach_ns)); + self.world.stats.borrow_mut().remove(&self.id); + self.result(Some(reason), end_mach_ns) + } + } + + impl MockCapture { + fn result(&self, reason: Option, end: Option) -> DisplayResult { + DisplayResult { + display: display(self.id, 100, 100, 0.0), + segment: self.segment, + scale: 1.0, + width_px: 100, + height_px: 100, + codec: crate::capture::Codec::Hevc, + file: video_file_name(self.id, self.segment), + frames_file: frames_file_name(self.id, self.segment), + video_t0_mach_ns: Some(u64::from(self.segment) * 1_000 + u64::from(self.id)), + video_end_mach_ns: end, + end_reason: reason, + frames: 10, + dropped: 0, + idle_callbacks: 0, + latency: LatencyStats::default(), + error: None, + } + } + } + + impl CaptureBackend for MockBackend { + fn active_displays(&self) -> eyre::Result> { + if self.0.probe_fails.get() { + eyre::bail!("window server is busy"); + } + + Ok(self.0.displays.borrow().clone()) + } + + fn start( + &self, + display: &DisplayDescriptor, + segment: u32, + _config: &CaptureConfig, + _dir: &Path, + ) -> eyre::Result> { + if self.0.failing.borrow().contains(&display.id) { + eyre::bail!("display not ready"); + } + + self.0.starts.borrow_mut().push((display.id, segment)); + let stats = Arc::new(CaptureStats::default()); + self.0.stats.borrow_mut().insert(display.id, stats.clone()); + Ok(Box::new(MockCapture { + id: display.id, + segment, + stats, + world: self.0.clone(), + })) + } + } + + fn manager(world: &Rc, follow: Follow) -> DisplayManager { + DisplayManager::new( + MockBackend(world.clone()), + CaptureConfig::default(), + PathBuf::from("/session"), + follow, + ) + } + + fn kinds(drafts: &[Draft]) -> Vec<&'static str> { + drafts.iter().map(|draft| draft.kind).collect() + } + + fn world_with(displays: Vec) -> Rc { + let world = Rc::new(World::default()); + *world.displays.borrow_mut() = displays; + world + } + + #[test] + fn every_present_display_starts_immediately() { + let world = world_with(vec![builtin(), external()]); + let mut manager = manager(&world, Follow::All); + + let drafts = manager.start(0).expect("start"); + + assert_eq!(manager.recording_count(), 2); + assert_eq!(*world.starts.borrow(), vec![(1, 1), (2, 1)]); + assert_eq!( + kinds(&drafts), + vec![kind::CAPTURE_STARTED, kind::CAPTURE_STARTED] + ); + } + + #[test] + fn a_display_connected_mid_recording_is_picked_up_after_it_settles() { + let world = world_with(vec![builtin()]); + let mut manager = manager(&world, Follow::All); + manager.start(0).expect("start"); + + // the dock is connected + world.displays.borrow_mut().push(external()); + let drafts = manager.poll(10 * SEC); + assert_eq!(kinds(&drafts), vec![kind::DISPLAY_CONNECTED]); + assert_eq!( + manager.recording_count(), + 1, + "not started yet: it is still settling" + ); + + assert!(manager.poll(10 * SEC + SETTLE_NS / 2).is_empty()); + let drafts = manager.poll(10 * SEC + SETTLE_NS); + assert_eq!(kinds(&drafts), vec![kind::CAPTURE_STARTED]); + assert_eq!(manager.recording_count(), 2); + assert_eq!(*world.starts.borrow(), vec![(1, 1), (2, 1)]); + } + + #[test] + fn a_disconnect_finalizes_only_that_display_and_keeps_the_others_untouched() { + let world = world_with(vec![builtin(), external()]); + let mut manager = manager(&world, Follow::All); + manager.start(0).expect("start"); + + // the external monitor is unplugged + world + .displays + .borrow_mut() + .retain(|display| display.id != 2); + let drafts = manager.poll(5 * SEC); + + assert_eq!(kinds(&drafts), vec![kind::DISPLAY_DISCONNECTED]); + assert_eq!( + *world.stops.borrow(), + vec![(2, 1, EndReason::DisplayDisconnected, None)] + ); + assert_eq!(manager.recording_count(), 1); + assert!( + world.stats.borrow().contains_key(&1), + "the built-in display's capture must keep running" + ); + } + + #[test] + fn a_reconnected_display_gets_a_new_segment_in_a_new_file() { + let world = world_with(vec![builtin(), external()]); + let mut manager = manager(&world, Follow::All); + manager.start(0).expect("start"); + + world + .displays + .borrow_mut() + .retain(|display| display.id != 2); + manager.poll(5 * SEC); + + world.displays.borrow_mut().push(external()); + manager.poll(20 * SEC); + manager.poll(20 * SEC + SETTLE_NS); + + assert_eq!(*world.starts.borrow(), vec![(1, 1), (2, 1), (2, 2)]); + + let results = manager.shutdown(); + let files: Vec<&str> = results.iter().map(|result| result.file.as_str()).collect(); + assert!( + files.contains(&"video-2.mp4") && files.contains(&"video-2-2.mp4"), + "{files:?}" + ); + assert_eq!(results.len(), 3); + } + + #[test] + fn a_dead_stream_ends_at_the_moment_it_died_and_is_restarted_after_a_delay() { + let world = world_with(vec![external()]); + let mut manager = manager(&world, Follow::All); + manager.start(0).expect("start"); + + // the machine slept: ScreenCaptureKit reports the stream stopped at t=30s + world.stats.borrow()[&2].set_error("stream stopped: display sleep".into(), 30 * SEC); + + let drafts = manager.poll(33 * SEC); + assert_eq!(kinds(&drafts), vec![kind::CAPTURE_STOPPED]); + assert_eq!( + drafts[0].mach_ns, + Some(30 * SEC), + "dated when it died, not when noticed" + ); + assert_eq!( + *world.stops.borrow(), + vec![(2, 1, EndReason::StreamError, Some(30 * SEC))] + ); + assert_eq!(manager.recording_count(), 0); + + // not restarted immediately... + assert!(manager.poll(33 * SEC + FIRST_BACKOFF_NS / 2).is_empty()); + // ...but after the backoff, as a new segment + let drafts = manager.poll(33 * SEC + FIRST_BACKOFF_NS); + assert_eq!(kinds(&drafts), vec![kind::CAPTURE_RECOVERED]); + assert_eq!(*world.starts.borrow(), vec![(2, 1), (2, 2)]); + } + + #[test] + fn a_failing_start_is_retried_with_backoff_and_reported_once() { + let world = world_with(vec![external()]); + world.failing.borrow_mut().insert(2); + let mut manager = manager(&world, Follow::All); + + let drafts = manager.start(0).expect("start"); + assert_eq!(kinds(&drafts), vec![kind::CAPTURE_FAILED]); + assert_eq!(manager.retrying(), vec![2]); + + // more failures: silent, with growing delays + let mut attempts = 0; + let mut now = 0; + for _ in 0..40 { + now += SEC; + let drafts = manager.poll(now); + assert!(drafts.is_empty(), "outage reported again: {drafts:?}"); + attempts += 1; + } + assert!(attempts > 0); + + // the display becomes ready (the screen was unlocked) + world.failing.borrow_mut().clear(); + let mut recovered = Vec::new(); + for _ in 0..12 { + now += SEC; + recovered.extend(manager.poll(now)); + } + + assert_eq!(kinds(&recovered), vec![kind::CAPTURE_RECOVERED]); + assert_eq!(manager.recording_count(), 1); + assert!(manager.retrying().is_empty()); + assert_eq!( + *world.starts.borrow(), + vec![(2, 1)], + "the failed attempts did not use up segment numbers" + ); + } + + #[test] + fn backoff_grows_and_is_capped() { + assert_eq!(backoff_ns(1), SEC); + assert_eq!(backoff_ns(2), 2 * SEC); + assert_eq!(backoff_ns(3), 4 * SEC); + assert_eq!(backoff_ns(4), 8 * SEC); + assert_eq!(backoff_ns(5), MAX_BACKOFF_NS); + assert_eq!(backoff_ns(500), MAX_BACKOFF_NS); + } + + #[test] + fn explicitly_selected_displays_ignore_the_others_but_recover_their_own() { + let world = world_with(vec![builtin(), external()]); + let mut manager = manager(&world, Follow::Only(BTreeSet::from([2]))); + manager.start(0).expect("start"); + assert_eq!(*world.starts.borrow(), vec![(2, 1)]); + + // a third monitor appears: not wanted + world + .displays + .borrow_mut() + .push(display(3, 1920, 1080, 3840.0)); + assert!(manager.poll(10 * SEC).is_empty()); + assert!(manager.poll(20 * SEC).is_empty()); + assert_eq!(*world.starts.borrow(), vec![(2, 1)]); + + // the selected one is unplugged and replugged: recorded again + world + .displays + .borrow_mut() + .retain(|display| display.id != 2); + manager.poll(30 * SEC); + world.displays.borrow_mut().push(external()); + manager.poll(40 * SEC); + manager.poll(40 * SEC + SETTLE_NS); + assert_eq!(*world.starts.borrow(), vec![(2, 1), (2, 2)]); + } + + #[test] + fn a_size_change_starts_a_new_segment_but_a_move_does_not() { + let world = world_with(vec![builtin()]); + let mut manager = manager(&world, Follow::All); + manager.start(0).expect("start"); + + // arrangement changed: same size, new position + *world.displays.borrow_mut() = vec![display(1, 1800, 1169, 0.0)]; + assert!(manager.poll(5 * SEC).is_empty()); + assert!(world.stops.borrow().is_empty()); + + // resolution changed: the old segment is finalized and the new one starts in the same + // poll, so there is no gap + *world.displays.borrow_mut() = vec![display(1, 1512, 982, 0.0)]; + let drafts = manager.poll(10 * SEC); + assert_eq!( + kinds(&drafts), + vec![kind::DISPLAY_RECONFIGURED, kind::CAPTURE_STARTED] + ); + assert_eq!(world.stops.borrow()[0].2, EndReason::Reconfigured); + assert_eq!(*world.starts.borrow(), vec![(1, 1), (1, 2)]); + assert_eq!(manager.recording_count(), 1); + } + + #[test] + fn an_unreadable_display_list_changes_nothing() { + let world = world_with(vec![builtin(), external()]); + let mut manager = manager(&world, Follow::All); + manager.start(0).expect("start"); + + world.probe_fails.set(true); + let drafts = manager.poll(5 * SEC); + assert_eq!(kinds(&drafts), vec![kind::RECORDER], "reported once"); + assert!(manager.poll(6 * SEC).is_empty(), "and not again"); + assert!( + world.stops.borrow().is_empty(), + "not being able to ask is not a disconnect" + ); + assert_eq!(manager.recording_count(), 2); + + world.probe_fails.set(false); + assert!(manager.poll(7 * SEC).is_empty()); + } + + #[test] + fn a_whole_sleep_wake_cycle_ends_and_restarts_every_display() { + let world = world_with(vec![builtin(), external()]); + let mut manager = manager(&world, Follow::All); + manager.start(0).expect("start"); + + // sleep: every display leaves at once + world.displays.borrow_mut().clear(); + let drafts = manager.poll(60 * SEC); + assert_eq!(drafts.len(), 2); + assert_eq!(manager.recording_count(), 0); + + // wake: both come back; they settle, then record as new segments + *world.displays.borrow_mut() = vec![builtin(), external()]; + manager.poll(3600 * SEC); + manager.poll(3600 * SEC + SETTLE_NS); + assert_eq!(manager.recording_count(), 2); + assert_eq!(*world.starts.borrow(), vec![(1, 1), (2, 1), (1, 2), (2, 2)]); + + let results = manager.shutdown(); + assert_eq!(results.len(), 4); + assert!( + results + .iter() + .filter(|result| result.segment == 1) + .all(|result| result.end_reason == Some(EndReason::DisplayDisconnected)), + "the pre-sleep segments were finalized when the displays left" + ); + } + + #[test] + fn nothing_is_lost_when_displays_flap_repeatedly() { + let world = world_with(vec![external()]); + let mut manager = manager(&world, Follow::All); + manager.start(0).expect("start"); + + let mut now = 0; + for _ in 0..5 { + now += 10 * SEC; + world.displays.borrow_mut().clear(); + manager.poll(now); + now += SEC; + *world.displays.borrow_mut() = vec![external()]; + manager.poll(now); + manager.poll(now + SETTLE_NS); + now += SETTLE_NS; + } + + let results = manager.shutdown(); + assert_eq!(results.len(), 6, "one segment per connection"); + let mut files: Vec<_> = results.iter().map(|result| result.file.clone()).collect(); + files.sort(); + files.dedup(); + assert_eq!(files.len(), 6, "no two segments share a file"); + // every segment was finalized, none abandoned + assert!(results.iter().all(|result| result.end_reason.is_some())); + } + + #[test] + fn checkpoints_include_live_segments_without_ending_them() { + let world = world_with(vec![builtin(), external()]); + let mut manager = manager(&world, Follow::All); + manager.start(0).expect("start"); + world + .displays + .borrow_mut() + .retain(|display| display.id != 2); + manager.poll(5 * SEC); + + let checkpoint = manager.checkpoint(); + assert_eq!(checkpoint.len(), 2); + assert_eq!( + checkpoint + .iter() + .filter(|result| result.end_reason.is_none()) + .count(), + 1, + "the live segment has no end yet" + ); + assert_eq!(manager.recording_count(), 1, "checkpointing did not end it"); + // the finished segment's frames plus the live capture's own counters + assert_eq!(manager.totals(), (10, 0)); + } + + #[test] + fn no_wanted_display_present_means_nothing_to_record_yet() { + let world = world_with(vec![]); + let mut manager = manager(&world, Follow::All); + manager.start(0).expect("start"); + assert_eq!(manager.recording_count(), 0); + assert!(manager.wanted_displays().expect("list").is_empty()); + + // a display shows up later and is recorded + world.displays.borrow_mut().push(external()); + manager.poll(SEC); + manager.poll(SEC + SETTLE_NS); + assert_eq!(manager.recording_count(), 1); + } + + #[test] + fn a_stalled_stream_is_replaced_and_ends_when_it_last_showed_life() { + let world = world_with(vec![external()]); + let mut manager = manager(&world, Follow::All); + manager.start(0).expect("start"); + + // healthy at 10s, then silent: no frames and no idle callbacks + world.signals.borrow_mut().insert(2, 10 * SEC); + assert!(manager.poll(10 * SEC + STALL_NS / 2).is_empty(), "not yet"); + + let drafts = manager.poll(10 * SEC + STALL_NS + SEC); + assert_eq!(kinds(&drafts), vec![kind::CAPTURE_STOPPED]); + assert!(drafts[0].detail.contains("stalled"), "{}", drafts[0].detail); + assert_eq!( + *world.stops.borrow(), + vec![(2, 1, EndReason::StreamError, Some(10 * SEC))], + "the segment ends when the stream last showed life, not when we noticed" + ); + + // and it is restarted as a new segment after the backoff + world.signals.borrow_mut().clear(); + manager.poll(10 * SEC + STALL_NS + SEC + FIRST_BACKOFF_NS); + assert_eq!(*world.starts.borrow(), vec![(2, 1), (2, 2)]); + } + + #[test] + fn a_healthy_idle_stream_is_never_mistaken_for_a_stalled_one() { + let world = world_with(vec![external()]); + let mut manager = manager(&world, Follow::All); + manager.start(0).expect("start"); + + // a static screen: no new frames for ten minutes, but idle callbacks keep arriving + let mut now = 0; + for _ in 0..600 { + now += SEC; + world.signals.borrow_mut().insert(2, now); + assert!(manager.poll(now).is_empty()); + } + + assert!(world.stops.borrow().is_empty()); + assert_eq!(manager.recording_count(), 1); + } + + #[test] + fn a_stream_that_never_signals_is_not_assumed_stalled_when_the_backend_cannot_tell() { + let world = world_with(vec![external()]); + let mut manager = manager(&world, Follow::All); + manager.start(0).expect("start"); + + assert!(manager.poll(3600 * SEC).is_empty()); + assert_eq!(manager.recording_count(), 1); + } + + #[test] + fn a_vanished_display_is_a_disconnect_even_if_its_stream_had_also_gone_quiet() { + let world = world_with(vec![external()]); + let mut manager = manager(&world, Follow::All); + manager.start(0).expect("start"); + world.signals.borrow_mut().insert(2, SEC); + + world.displays.borrow_mut().clear(); + let drafts = manager.poll(60 * SEC); + assert_eq!(kinds(&drafts), vec![kind::DISPLAY_DISCONNECTED]); + } + + #[test] + fn every_live_capture_is_offered_a_keepalive_on_every_poll() { + let world = world_with(vec![builtin(), external()]); + let mut manager = manager(&world, Follow::All); + manager.start(0).expect("start"); + world.keepalives.borrow_mut().clear(); + + manager.poll(SEC); + manager.poll(2 * SEC); + let calls = world.keepalives.borrow().clone(); + assert_eq!(calls.iter().filter(|id| **id == 1).count(), 2); + assert_eq!(calls.iter().filter(|id| **id == 2).count(), 2); + } + + #[test] + fn waking_from_sleep_replaces_every_capture_after_a_settle_delay() { + let world = world_with(vec![builtin(), external()]); + let mut manager = manager(&world, Follow::All); + manager.start(0).expect("start"); + + // the machine slept for an hour and woke with the same displays attached + let drafts = manager.restart_all(100 * SEC, EndReason::Resumed); + assert_eq!(drafts.len(), 2); + assert_eq!(manager.recording_count(), 0); + assert!( + world + .stops + .borrow() + .iter() + .all(|stop| stop.2 == EndReason::Resumed) + ); + + assert!(manager.poll(100 * SEC).is_empty(), "still settling"); + let drafts = manager.poll(100 * SEC + SETTLE_NS); + assert_eq!( + kinds(&drafts), + vec![kind::CAPTURE_STARTED, kind::CAPTURE_STARTED] + ); + assert_eq!(*world.starts.borrow(), vec![(1, 1), (2, 1), (1, 2), (2, 2)]); + } + + #[test] + fn restarting_after_sleep_does_not_resurrect_a_display_that_did_not_come_back() { + let world = world_with(vec![builtin(), external()]); + let mut manager = manager(&world, Follow::All); + manager.start(0).expect("start"); + + manager.restart_all(100 * SEC, EndReason::Resumed); + // the external monitor was unplugged while the laptop slept + world + .displays + .borrow_mut() + .retain(|display| display.id != 2); + + manager.poll(100 * SEC + SETTLE_NS); + assert_eq!(manager.recording_count(), 1); + assert_eq!(*world.starts.borrow(), vec![(1, 1), (2, 1), (1, 2)]); + } + + #[test] + fn a_sleeping_display_ends_its_segment_at_once_and_returns_as_a_new_one_when_it_wakes() { + let world = world_with(vec![builtin(), external()]); + let mut manager = manager(&world, Follow::All); + manager.start(0).expect("start"); + + // the built-in display goes to sleep (still attached, but dark) + *world.displays.borrow_mut() = vec![asleep(builtin()), external()]; + let drafts = manager.poll(30 * SEC); + assert_eq!(kinds(&drafts), vec![kind::DISPLAY_ASLEEP]); + assert_eq!( + *world.stops.borrow(), + vec![(1, 1, EndReason::DisplayAsleep, None)] + ); + assert_eq!( + manager.recording_count(), + 1, + "the other display is untouched" + ); + + // nothing is attempted, reported or retried while it sleeps + for step in 1..30 { + assert!(manager.poll((30 + step) * SEC).is_empty()); + } + assert_eq!(*world.starts.borrow(), vec![(1, 1), (2, 1)]); + assert!(manager.retrying().is_empty()); + + // it wakes: reported as available again (not as a new display), then recorded afresh + *world.displays.borrow_mut() = vec![builtin(), external()]; + let drafts = manager.poll(100 * SEC); + assert_eq!(kinds(&drafts), vec![kind::DISPLAY_AWAKE]); + let drafts = manager.poll(100 * SEC + SETTLE_NS); + assert_eq!(kinds(&drafts), vec![kind::CAPTURE_STARTED]); + assert_eq!(*world.starts.borrow(), vec![(1, 1), (2, 1), (1, 2)]); + } + + #[test] + fn a_display_that_is_asleep_when_recording_begins_waits_to_be_recorded() { + let world = world_with(vec![asleep(builtin()), external()]); + let mut manager = manager(&world, Follow::All); + manager.start(0).expect("start"); + assert_eq!(*world.starts.borrow(), vec![(2, 1)]); + + *world.displays.borrow_mut() = vec![builtin(), external()]; + manager.poll(10 * SEC); + manager.poll(10 * SEC + SETTLE_NS); + assert_eq!(*world.starts.borrow(), vec![(2, 1), (1, 1)]); + } + + #[test] + fn a_reconnected_display_is_reported_as_available_again_not_as_new() { + let world = world_with(vec![builtin(), external()]); + let mut manager = manager(&world, Follow::All); + manager.start(0).expect("start"); + + world + .displays + .borrow_mut() + .retain(|display| display.id != 2); + manager.poll(5 * SEC); + world.displays.borrow_mut().push(external()); + let drafts = manager.poll(9 * SEC); + assert_eq!(kinds(&drafts), vec![kind::DISPLAY_AWAKE]); + assert!( + drafts[0].detail.contains("available again"), + "{}", + drafts[0].detail + ); + + // a genuinely new display is still "connected" (and the returning one, settled by now, + // has started recording) + world + .displays + .borrow_mut() + .push(display(7, 1920, 1080, 3840.0)); + let drafts = manager.poll(20 * SEC); + let connected: Vec<_> = drafts + .iter() + .filter(|draft| draft.kind == kind::DISPLAY_CONNECTED) + .collect(); + assert_eq!(connected.len(), 1); + assert_eq!(connected[0].display, Some(7)); + assert!(kinds(&drafts).contains(&kind::CAPTURE_STARTED)); + } +} diff --git a/komorebi-recorder/src/capture/mod.rs b/komorebi-recorder/src/capture/mod.rs new file mode 100644 index 00000000..acc6c173 --- /dev/null +++ b/komorebi-recorder/src/capture/mod.rs @@ -0,0 +1,1223 @@ +//! Screen capture: the plain data that describes a recording of a display, the two traits that +//! produce one, and the policy that keeps every display recording while the machine changes under +//! it. +//! +//! The unit of recording is a **segment**: one display, one contiguous span of time, one file. A +//! display that is unplugged and plugged back in, whose stream dies, or that changes size produces +//! several segments, each a complete, playable file. A segment is two files in the session +//! directory, named by [`video_file_name`] and [`frames_file_name`]: +//! +//! - `video-[-].mp4`, a fragmented MP4, so that a recorder that is killed leaves a +//! file that plays up to its last fragment; +//! - `frames-[-].csv`, when each frame was shown, on the master clock. +//! +//! [`DisplayResult`] describes one segment. The session manifest stores them (see +//! [`crate::session::DisplayEntry`]). +//! +//! # Where things are +//! +//! | item | role | +//! |---|---| +//! | [`CaptureBackend`] | Where displays and captures come from: lists the displays that exist and starts a capture of one. | +//! | [`RunningCapture`] | One segment being recorded: its counters, a snapshot of it, a keep-alive hook and `stop`. | +//! | [`manager`] | [`DisplayManager`](manager::DisplayManager), which keeps every wanted display recording through hotplug, sleep, lock and stalled streams. It uses only the two traits above. | +//! | [`segment`] | The rules of one segment that do not depend on the OS: where a frame goes in the file's time, the frame log, when an idle display needs a frame repeated, how to size the encoding. | +//! | `sck` | The macOS backend: `ScreenCaptureKit` into an `AVAssetWriter`. Exists on macOS only. | +//! | `wgc` | The Windows backend: Windows.Graphics.Capture into a Media Foundation sink writer. Exists on Windows only. | +//! +//! The two backends are the only code here that calls a capture API. Everything else is plain Rust, +//! which is what makes display hotplug, sleep and wake testable: [`manager`] is tested against a scripted +//! [`CaptureBackend`] (docking, undocking, flapping and sleeping displays, stalled streams), since +//! real hardware cannot be unplugged from a test. +//! +//! # The host axis +//! +//! The rest of the crate does not name a backend. [`Capture`] is the host's backend +//! ([`crate::platform::capture::Backend`]) or [`NoDisplays`], for a recording made without video. +//! The two real backends differ in what the OS gives them: +//! +//! | | macOS | Windows | +//! |---|---|---| +//! | Capture API | `ScreenCaptureKit` | Windows.Graphics.Capture | +//! | File writer | `AVAssetWriter` | Media Foundation sink writer | +//! | Codec | HEVC or H.264 | H.264 only: [`Codec::Hevc`] is refused when a capture starts | +//! | Frame timestamps | the frame's presentation time, on `mach_absolute_time` | the frame's `SystemRelativeTime`, on the performance counter | +//! | A screen that is not changing | idle callbacks, about thirty a second, which are the only sign of life | no callbacks at all | +//! | Stall detection ([`RunningCapture::last_signal_mach_ns`]) | on | off: silence is not a sign of death | +//! +//! # Time +//! +//! Every `*_mach_ns` value in this module is a time on the master clock (see [`crate::clock`]): +//! nanoseconds on the host's monotonic clock. The `mach_` spelling is historical; the session +//! manifest's `host.clock` records which clock it was (`mach`, `qpc` or `std`). +//! +//! # What this module does not do +//! +//! It does not read recorded video back (`av` does), recover a session whose recorder was killed +//! (`recovery` does, from the frame log and the file), or notice that the machine slept (`power` +//! and the `recorder` loop do, and then ask [`manager::DisplayManager::restart_all`] to replace the +//! captures). + +use color_eyre::eyre; +use parking_lot::Mutex; +use serde::Deserialize; +use serde::Serialize; +use std::path::Path; +use std::sync::Arc; +use std::sync::atomic::AtomicU64; +use std::sync::atomic::Ordering; + +pub mod manager; +#[cfg(target_os = "macos")] +pub mod sck; +pub mod segment; +#[cfg(windows)] +pub mod wgc; + +/// The video codec a segment is encoded with. +/// +/// The text forms are `hevc` and `h264`. [`Display`](std::fmt::Display) writes them, `session.json` +/// stores them, and the `--codec` option of `record` accepts them. +/// +/// The default is [`Codec::Hevc`] on a Mac and [`Codec::H264`] on Windows. The macOS backend +/// records either. The Windows backend records only H.264 and refuses [`Codec::Hevc`] when a +/// capture starts. +/// +/// # Example +/// +/// ```rust +/// use clap::ValueEnum; +/// use komorebi_recorder::capture::Codec; +/// +/// // `Display`, `serde` and the command line agree on the spelling. +/// assert_eq!(Codec::Hevc.to_string(), "hevc"); +/// assert_eq!(serde_json::to_string(&Codec::H264)?, r#""h264""#); +/// assert_eq!(serde_json::from_str::(r#""hevc""#)?, Codec::Hevc); +/// let possible = Codec::H264.to_possible_value().expect("not skipped"); +/// assert_eq!(possible.get_name(), "h264"); +/// +/// // The default depends on the host the crate was built for. +/// let expected = if cfg!(windows) { Codec::H264 } else { Codec::Hevc }; +/// assert_eq!(Codec::default(), expected); +/// # Ok::<(), Box>(()) +/// ``` +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, clap::ValueEnum)] +#[serde(rename_all = "lowercase")] +pub enum Codec { + /// HEVC (H.265). Recorded on macOS only. Playing it back on Windows needs a decoder that + /// Windows does not always have (the HEVC Video Extensions). + Hevc, + /// H.264. Recorded on both hosts, and decoded by Media Foundation everywhere. + H264, +} + +impl Default for Codec { + /// HEVC on a Mac. H.264 on Windows: Media Foundation decodes it everywhere, HEVC needs the + /// HEVC Video Extensions to be installed, and its MP4 writer cannot write HEVC without a + /// hand-built sample description. + fn default() -> Self { + if cfg!(windows) { + Codec::H264 + } else { + Codec::Hevc + } + } +} + +impl std::fmt::Display for Codec { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Codec::Hevc => f.write_str("hevc"), + Codec::H264 => f.write_str("h264"), + } + } +} + +/// How every segment of a recording is captured: the frame rate, the resolution and the codec. +/// +/// One configuration applies to every display and every segment of a recording. +/// [`DisplayManager::new`](manager::DisplayManager::new) takes it once and hands it to +/// [`CaptureBackend::start`] for each segment. +/// +/// The default is 30 frames a second, not retina, and [`Codec::default`]. +/// +/// # Example +/// +/// ```rust +/// use komorebi_recorder::capture::{CaptureConfig, Codec}; +/// +/// let config = CaptureConfig::default(); +/// assert_eq!(config.fps, 30); +/// assert!(!config.retina); +/// assert_eq!(config.codec, Codec::default()); +/// ``` +#[derive(Debug, Clone)] +pub struct CaptureConfig { + /// The frame rate the capture is configured for. `0` is treated as `1`. + /// + /// On a Mac this is an upper bound: `ScreenCaptureKit` is asked for no more than `fps` frames a + /// second, and emits frames only when the content changes. On Windows it is the nominal rate + /// declared to the encoder. It sets the bitrate (see [`segment::h264_bitrate`]), the key frame + /// spacing and the duration given to the last frame, and does not limit how often frames + /// arrive. + pub fps: u32, + /// Whether to capture at native pixel resolution rather than logical points. + /// + /// Read by the macOS backend only: a display `width_points` wide is captured at + /// `width_points * scale` pixels, where `scale` is the point-to-pixel factor `ScreenCaptureKit` + /// reports for it (see [`DisplayResult::scale`]). Windows frames are always native pixels. + pub retina: bool, + /// The codec to encode with. See [`Codec`] for what each host accepts. + pub codec: Codec, +} + +impl Default for CaptureConfig { + fn default() -> Self { + Self { + fps: 30, + retina: false, + codec: Codec::default(), + } + } +} + +/// A display that can be captured, as the OS described it at one moment. +/// +/// A descriptor is a snapshot. [`CaptureBackend::active_displays`] returns a fresh list each time +/// it is asked, and [`DisplayManager`](manager::DisplayManager) compares the list with the +/// descriptor each running capture started with to find displays that left, fell asleep or changed +/// size. Equality compares every field, position included; [`DisplayDescriptor::size_differs`] +/// compares only what invalidates a capture. +/// +/// # Example +/// +/// A descriptor read from a manifest that has no `asleep` or `scale` fields gets the defaults: +/// +/// ```rust +/// use komorebi_recorder::capture::DisplayDescriptor; +/// +/// let display: DisplayDescriptor = serde_json::from_str( +/// r#"{"id": 2, "frame": [0.0, 0.0, 3840.0, 1620.0], "width_points": 3840, "height_points": 1620}"#, +/// )?; +/// assert!(!display.asleep); +/// assert_eq!(display.scale, 1.0); +/// # Ok::<(), Box>(()) +/// ``` +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct DisplayDescriptor { + /// The OS's identifier for the display: the `CGDirectDisplayID` on a Mac, and on Windows the + /// `HMONITOR` handle value (truncated to 32 bits), which is also the id komorebi for Windows + /// gives the monitor. [`DisplayManager`](manager::DisplayManager) treats an id that reappears + /// after it was gone as the same display returning. + pub id: u32, + /// `[x, y, width, height]` of the display in the global desktop's coordinates: logical points + /// on a Mac, physical pixels on Windows (the left and top of its monitor rectangle, then its + /// width and height). + pub frame: [f64; 4], + /// The width of `frame`, rounded to a whole number. + pub width_points: i64, + /// The height of `frame`, rounded to a whole number. + pub height_points: i64, + /// Attached but asleep. A sleeping display cannot be captured and its stream stalls, so it is + /// treated as absent until it wakes. The Windows host always reports `false`. + #[serde(default)] + pub asleep: bool, + /// The display's scaling factor. On a Mac, how many pixels the display has for each point of + /// `frame`: the display mode's pixel width over its width in points, which is `2.0` on a + /// Retina display. On Windows, the display's effective DPI over 96 (`1.5` at 150% scaling); + /// `frame` is already in physical pixels there, so it is not a factor to convert `frame` by. + /// Absent in a manifest means `1.0`. + #[serde(default = "unit_scale")] + pub scale: f64, +} + +fn unit_scale() -> f64 { + 1.0 +} + +impl DisplayDescriptor { + /// Returns `true` if and only if the display's size differs from `other`'s. + /// + /// A change of size (a resolution change, or the same display id coming back as a different + /// monitor) invalidates a running capture, whereas a change of position (rearranging displays) + /// does not. Only `width_points` and `height_points` are compared. + /// + /// # Example + /// + /// ```rust + /// use komorebi_recorder::capture::DisplayDescriptor; + /// + /// let display = |x: f64, width: i64, height: i64| DisplayDescriptor { + /// id: 1, + /// frame: [x, 0.0, width as f64, height as f64], + /// width_points: width, + /// height_points: height, + /// asleep: false, + /// scale: 1.0, + /// }; + /// + /// // Rearranged: same size, new position. + /// assert!(!display(0.0, 1800, 1169).size_differs(&display(-1800.0, 1800, 1169))); + /// // A resolution change. + /// assert!(display(0.0, 1800, 1169).size_differs(&display(0.0, 1512, 982))); + /// ``` + #[must_use] + pub fn size_differs(&self, other: &Self) -> bool { + self.width_points != other.width_points || self.height_points != other.height_points + } +} + +/// How far a segment's frame timestamps lag behind the moment each frame was received. +/// +/// Each frame is stamped with the time the display produced it and each callback records when it +/// arrived. The difference is `received - stamped`, in nanoseconds: positive when the stamp is +/// earlier than the arrival (the usual case), negative when the stamp is ahead of it. If the +/// timestamps were not on the master clock the numbers would be off by seconds or more, so +/// [`LatencyStats::looks_like_host_clock`] doubles as the clock-unification check. +/// +/// Built by [`segment::Latency`], which both the backends use, and stored in +/// [`DisplayResult::latency`]. A segment with no frames has the default, all zeros. +#[derive( + Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize, schemars::JsonSchema, +)] +pub struct LatencyStats { + /// How many frames were measured. + pub samples: u64, + /// The smallest `received - stamped`, in nanoseconds. Negative when a frame was stamped ahead + /// of its arrival. + pub min_ns: i64, + /// The largest `received - stamped`, in nanoseconds. + pub max_ns: i64, + /// The mean `received - stamped`, in nanoseconds, truncated toward zero. + pub mean_ns: i64, +} + +impl LatencyStats { + /// Returns `true` if and only if the measured latencies are plausible for timestamps on the + /// master clock. + /// + /// That is the case when there are no samples (nothing to doubt), or when no frame was + /// stamped more than 100 ms ahead of its arrival (`min_ns >= -100 ms`) and none arrived 2 s or + /// more after its stamp (`max_ns < 2 s`). The bounds only have to separate "the same clock" + /// from "a different clock", which is off by seconds or more: on a Mac with real displays the + /// measured range was about -7.5 ms to 196 ms, the late end being the first frame while the + /// encoder spins up. + /// + /// # Example + /// + /// ```rust + /// use komorebi_recorder::capture::LatencyStats; + /// + /// // Measured on a real display. + /// let measured = LatencyStats { + /// samples: 229, + /// min_ns: -7_506_167, + /// max_ns: 196_015_958, + /// mean_ns: 6_524_913, + /// }; + /// assert!(measured.looks_like_host_clock()); + /// + /// // Stamps from another clock, here one an hour off, are not plausible. + /// let wrong = LatencyStats { + /// samples: 10, + /// min_ns: 3_600_000_000_000, + /// max_ns: 3_600_100_000_000, + /// mean_ns: 3_600_050_000_000, + /// }; + /// assert!(!wrong.looks_like_host_clock()); + /// + /// // No frames, nothing to doubt. + /// assert!(LatencyStats::default().looks_like_host_clock()); + /// ``` + #[must_use] + pub fn looks_like_host_clock(&self) -> bool { + // Frames are stamped at (or marginally ahead of) the display refresh that produced them: + // measured -7.5 ms at the early end and ~200 ms at the late end (the first frame, while + // the encoder spins up). A *different* clock would be off by seconds or more, so these + // bounds only have to separate "same clock" from "not the same clock". + self.samples == 0 || (self.min_ns >= -100_000_000 && self.max_ns < 2_000_000_000) + } +} + +/// Why a capture segment ended. +/// +/// Stored in [`DisplayResult::end_reason`] and written to `session.json` in kebab-case +/// (`display-disconnected`, `stream-error`, ...). [`Display`](std::fmt::Display) writes a phrase +/// for people (`display disconnected`), which the incident log uses. +/// +/// # Example +/// +/// ```rust +/// use komorebi_recorder::capture::EndReason; +/// +/// assert_eq!( +/// serde_json::to_string(&EndReason::DisplayDisconnected)?, +/// r#""display-disconnected""# +/// ); +/// assert_eq!(EndReason::Resumed.to_string(), "machine resumed from sleep"); +/// # Ok::<(), Box>(()) +/// ``` +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum EndReason { + /// The recording was stopped normally. + Stopped, + /// The display went away (unplugged, undocked, lid closed): it was missing from the display + /// list. + DisplayDisconnected, + /// The display went to sleep; a new segment starts when it wakes. + DisplayAsleep, + /// The capture reported an error, or showed no sign of life for too long (see + /// [`RunningCapture::last_signal_mach_ns`]). On a Mac that is `ScreenCaptureKit` stopping the + /// stream (sleep, lock, a display reconfiguration, ...). On Windows it is the display's capture + /// item closing, the GPU being reset or removed, a frame of a different size, or the encoder + /// rejecting a frame. + StreamError, + /// The display's size changed, so the segment was closed and a new one begun. + Reconfigured, + /// The machine slept and woke. Capture streams are not trusted across a sleep (they can stall + /// without reporting anything), so each is replaced with a fresh one by + /// [`manager::DisplayManager::restart_all`]. + Resumed, + /// The recorder died before finalizing the segment, which was recovered from the fragmented + /// file afterwards (`recovery` sets this; no capture does). + Interrupted, +} + +impl std::fmt::Display for EndReason { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(match self { + EndReason::Stopped => "stopped", + EndReason::DisplayDisconnected => "display disconnected", + EndReason::DisplayAsleep => "display asleep", + EndReason::StreamError => "stream error", + EndReason::Reconfigured => "display reconfigured", + EndReason::Resumed => "machine resumed from sleep", + EndReason::Interrupted => "interrupted", + }) + } +} + +/// Returns the file name of a segment's video. +/// +/// The first segment of a display (`segment` 0 or 1) has the plain name `video-.mp4`, so a +/// session with one segment per display has no numbers in its file names. Later segments are +/// `video--.mp4`. +/// +/// # Example +/// +/// ```rust +/// use komorebi_recorder::capture::video_file_name; +/// +/// assert_eq!(video_file_name(2, 1), "video-2.mp4"); +/// assert_eq!(video_file_name(2, 3), "video-2-3.mp4"); +/// +/// // The display id and the segment number cannot be confused for one another. +/// assert_ne!(video_file_name(1, 2), video_file_name(12, 1)); +/// ``` +#[must_use] +pub fn video_file_name(display: u32, segment: u32) -> String { + if segment <= 1 { + format!("video-{display}.mp4") + } else { + format!("video-{display}-{segment}.mp4") + } +} + +/// Returns the file name of a segment's frame timestamp log. +/// +/// It is named like the segment's video (see [`video_file_name`]), with `frames-` and `.csv` in +/// place of `video-` and `.mp4`. +/// +/// # Example +/// +/// ```rust +/// use komorebi_recorder::capture::frames_file_name; +/// +/// assert_eq!(frames_file_name(2, 1), "frames-2.csv"); +/// assert_eq!(frames_file_name(2, 3), "frames-2-3.csv"); +/// ``` +#[must_use] +pub fn frames_file_name(display: u32, segment: u32) -> String { + if segment <= 1 { + format!("frames-{display}.csv") + } else { + format!("frames-{display}-{segment}.csv") + } +} + +/// One contiguous recording of one display: a display that is unplugged and replugged, or whose +/// stream dies, produces several of these, each a complete, playable file. +/// +/// A result is returned by [`RunningCapture::stop`] when the segment is over and by +/// [`RunningCapture::partial`] while it is still running. A running segment has no +/// [`end_reason`](DisplayResult::end_reason) and no +/// [`video_end_mach_ns`](DisplayResult::video_end_mach_ns). The session manifest stores every +/// result (see [`crate::session::DisplayEntry`]). +/// +/// The serialized form is what `session.json` stores for each segment. `segment`, +/// `video_end_mach_ns`, `end_reason` and `idle_callbacks` may be absent when it is read, and take +/// their defaults. +/// +/// # Example +/// +/// ```rust +/// use komorebi_recorder::capture::DisplayResult; +/// +/// // A result with none of the optional fields: segment 1, still open, not idle-counted. +/// let result: DisplayResult = serde_json::from_str(r#"{ +/// "display": {"id": 2, "frame": [0.0, 0.0, 3840.0, 1620.0], +/// "width_points": 3840, "height_points": 1620}, +/// "scale": 1.0, "width_px": 3840, "height_px": 1620, "codec": "hevc", +/// "file": "video-2.mp4", "frames_file": "frames-2.csv", +/// "video_t0_mach_ns": 5, "frames": 3, "dropped": 0, +/// "latency": {"samples": 0, "min_ns": 0, "max_ns": 0, "mean_ns": 0}, +/// "error": null +/// }"#)?; +/// assert_eq!(result.segment, 1); +/// assert_eq!(result.video_end_mach_ns, None); +/// assert_eq!(result.end_reason, None); +/// assert_eq!(result.idle_callbacks, 0); +/// # Ok::<(), Box>(()) +/// ``` +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct DisplayResult { + /// The display as the OS described it when the segment began. + pub display: DisplayDescriptor, + /// The segment's number: 1-based, per display. Segment `n` of display `d` is the file + /// [`video_file_name`]`(d, n)`. + #[serde(default = "first_segment")] + pub segment: u32, + /// How many pixels the video has for each unit of `display.frame`. `1.0`, except for a Mac + /// recording made at native resolution (see [`CaptureConfig::retina`]), where it is the + /// point-to-pixel factor `ScreenCaptureKit` reports. Always `1.0` on Windows. + pub scale: f64, + /// The width of the video in pixels. Both backends round it to an even number, as video codecs + /// want. + pub width_px: usize, + /// The height of the video in pixels, rounded to an even number as `width_px` is. + pub height_px: usize, + /// The codec the file is encoded with. + pub codec: Codec, + /// The video's file name, relative to the session directory (see [`video_file_name`]). + pub file: String, + /// The frame log's file name, relative to the session directory (see [`frames_file_name`]). + pub frames_file: String, + /// Master-clock time of the first frame in the file, if any frame was received. This is the + /// instant that is time zero of the file. `None` means no frame arrived. + pub video_t0_mach_ns: Option, + /// Master-clock time this segment ended (frames are variable-rate, so the last frame can be + /// earlier). `None` while it is still being recorded. + #[serde(default)] + pub video_end_mach_ns: Option, + /// Why the segment ended. `None` while it is still being recorded. + #[serde(default)] + pub end_reason: Option, + /// How many real frames were written. A repeat of the last frame of a still display (which + /// keeps the file moving) is not counted. + pub frames: u64, + /// How many frames were discarded because the writer could not take them. Counted by the + /// macOS backend; the Windows backend does not count drops and always reports `0`. + pub dropped: u64, + /// Callbacks that carried no new content. A healthy stream on a static screen still makes + /// these, so a segment with frames but none of these, or vice versa, tells you which way a + /// "frozen video" report went wrong. Always `0` on Windows, which does not call back without + /// a new frame. + #[serde(default)] + pub idle_callbacks: u64, + /// How far the frames' timestamps lagged behind their arrival (the clock check). + pub latency: LatencyStats, + /// The error that affected the segment, if any: the first error the capture reported, or a + /// failure to stop the capture or finalize the file. `"no frames were captured"` for a segment + /// that never received a frame. `None` for a segment that went well, and for one still being + /// recorded that has had no error yet. + pub error: Option, +} + +fn first_segment() -> u32 { + 1 +} + +/// Live counters, readable from any thread while a capture is running. +/// +/// A [`RunningCapture`] shares one with the callbacks that feed it and hands it out through +/// [`RunningCapture::stats`]. [`manager::DisplayManager`] reads the error to tell that a stream +/// died and the counters for its [totals](manager::DisplayManager::totals). A time of `0` means +/// "not yet" in the atomic fields. The accessor of the same name turns that into an `Option`, and +/// is the one to prefer. +/// +/// Only the first error is kept (see [`CaptureStats::set_error`]). +/// +/// # Example +/// +/// ```rust +/// use komorebi_recorder::capture::CaptureStats; +/// use std::sync::atomic::Ordering; +/// +/// let stats = CaptureStats::default(); +/// assert_eq!(stats.first_pts_mach_ns(), None); +/// +/// // A capture callback records a frame. +/// stats.first_pts_mach_ns.store(5_000, Ordering::Relaxed); +/// stats.frames.fetch_add(1, Ordering::Relaxed); +/// assert_eq!(stats.first_pts_mach_ns(), Some(5_000)); +/// +/// // Two errors: the second is a consequence of the first, and is dropped. +/// stats.set_error("stream stopped".into(), 9_000); +/// stats.set_error("could not append frame".into(), 9_500); +/// assert_eq!(stats.error().as_deref(), Some("stream stopped")); +/// assert_eq!(stats.error_at_mach_ns(), Some(9_000)); +/// ``` +#[derive(Debug, Default)] +pub struct CaptureStats { + /// Real frames written so far. + pub frames: AtomicU64, + /// Frames discarded because the writer could not take them. Only the macOS backend counts + /// these. + pub dropped: AtomicU64, + /// The master-clock time of the first frame, `0` until the first frame arrives. See + /// [`CaptureStats::first_pts_mach_ns()`]. + pub first_pts_mach_ns: AtomicU64, + /// The master-clock time of the last real frame written, after any adjustment to keep it + /// later than what came before, `0` until a frame arrives. See + /// [`CaptureStats::last_pts_mach_ns()`]. + pub last_pts_mach_ns: AtomicU64, + /// Callbacks of any kind, including "idle" ones that carry no new content. `ScreenCaptureKit` + /// keeps calling while a stream is healthy even when the screen is not changing, which makes + /// this the liveness signal a static display otherwise lacks. + pub callbacks: AtomicU64, + /// Callbacks that reported no new content (status idle/blank/suspended). Only the macOS + /// backend counts these. + pub idle_callbacks: AtomicU64, + /// When the last callback of any kind arrived (`0` for none yet), on the master clock. See + /// [`CaptureStats::last_callback_mach_ns()`]. + pub last_callback_mach_ns: AtomicU64, + /// When the first error was reported (`0` for none), on the master clock, so a dead stream's + /// segment can end at the moment it died rather than whenever it was noticed. See + /// [`CaptureStats::error_at_mach_ns()`]. + pub error_at_mach_ns: AtomicU64, + error: Mutex>, +} + +impl CaptureStats { + /// Returns the master-clock time of the first frame, or `None` before one has arrived. + #[must_use] + pub fn first_pts_mach_ns(&self) -> Option { + match self.first_pts_mach_ns.load(Ordering::Relaxed) { + 0 => None, + ns => Some(ns), + } + } + + /// Returns when the last callback of any kind arrived, or `None` if none has. + #[must_use] + pub fn last_callback_mach_ns(&self) -> Option { + match self.last_callback_mach_ns.load(Ordering::Relaxed) { + 0 => None, + ns => Some(ns), + } + } + + /// Returns the master-clock time of the last real frame written, or `None` before one has + /// arrived. + #[must_use] + pub fn last_pts_mach_ns(&self) -> Option { + match self.last_pts_mach_ns.load(Ordering::Relaxed) { + 0 => None, + ns => Some(ns), + } + } + + /// Returns the first error reported, or `None` if there has been none. + #[must_use] + pub fn error(&self) -> Option { + self.error.lock().clone() + } + + /// Returns when the first error was reported, on the master clock, or `None` if there has + /// been none. + #[must_use] + pub fn error_at_mach_ns(&self) -> Option { + match self.error_at_mach_ns.load(Ordering::Relaxed) { + 0 => None, + ns => Some(ns), + } + } + + /// Records the first error and the time it happened; later ones are consequences of it, and + /// are ignored. + pub fn set_error(&self, message: String, at_mach_ns: u64) { + let mut error = self.error.lock(); + if error.is_none() { + *error = Some(message); + self.error_at_mach_ns.store(at_mach_ns, Ordering::Relaxed); + } + } +} + +/// A capture of one display that is in progress: one segment, from its first frame until it is +/// stopped. +/// +/// A [`CaptureBackend`] returns one from [`CaptureBackend::start`]. [`manager::DisplayManager`] +/// keeps it while the display is recording, calls [`keepalive`](RunningCapture::keepalive) on every +/// poll, reads its [`stats`](RunningCapture::stats) and +/// [`last_signal_mach_ns`](RunningCapture::last_signal_mach_ns) to tell a dead or stalled stream, +/// and ends it with [`stop`](RunningCapture::stop), which consumes it. +/// +/// # What an implementor writes +/// +/// | method | required | what it is for | +/// |---|---|---| +/// | [`display_id`](RunningCapture::display_id), [`segment`](RunningCapture::segment) | yes | which display and which segment this is | +/// | [`stats`](RunningCapture::stats) | yes | the live counters and the first error, shared with the capture's callbacks | +/// | [`partial`](RunningCapture::partial) | yes | a snapshot of the segment so far, for manifest checkpoints | +/// | [`stop`](RunningCapture::stop) | yes | ends the segment and finalizes the file | +/// | [`last_signal_mach_ns`](RunningCapture::last_signal_mach_ns) | no: `None` | the liveness signal; without one a stalled capture is never noticed | +/// | [`keepalive`](RunningCapture::keepalive) | no: does nothing | repeats the last frame of an idle display | +/// +/// The compiler enforces the required methods: an implementation that leaves one out does not +/// compile, here for `stop`. +/// +/// ```rust,compile_fail,E0046 +/// # use komorebi_recorder::capture::{CaptureStats, DisplayResult, RunningCapture}; +/// # use std::sync::Arc; +/// struct Recording; +/// +/// impl RunningCapture for Recording { +/// fn display_id(&self) -> u32 { +/// 1 +/// } +/// fn segment(&self) -> u32 { +/// 1 +/// } +/// fn stats(&self) -> Arc { +/// Arc::default() +/// } +/// fn partial(&self) -> DisplayResult { +/// unimplemented!() +/// } +/// // no `stop` +/// } +/// ``` +/// +/// # Example +/// +/// A capture that writes nothing, to show the whole contract. The two provided methods are left as +/// they are, so it reports no sign of life and has nothing to keep alive: +/// +/// ```rust +/// use komorebi_recorder::capture::{ +/// CaptureStats, Codec, DisplayDescriptor, DisplayResult, EndReason, LatencyStats, +/// RunningCapture, +/// }; +/// use std::sync::Arc; +/// +/// struct Recording { +/// display: DisplayDescriptor, +/// segment: u32, +/// stats: Arc, +/// } +/// # impl Recording { +/// # fn result(&self, reason: Option, end: Option) -> DisplayResult { +/// # use komorebi_recorder::capture::{frames_file_name, video_file_name}; +/// # DisplayResult { +/// # display: self.display.clone(), +/// # segment: self.segment, +/// # scale: 1.0, +/// # width_px: 1920, +/// # height_px: 1080, +/// # codec: Codec::H264, +/// # file: video_file_name(self.display.id, self.segment), +/// # frames_file: frames_file_name(self.display.id, self.segment), +/// # video_t0_mach_ns: self.stats.first_pts_mach_ns(), +/// # video_end_mach_ns: end, +/// # end_reason: reason, +/// # frames: 0, +/// # dropped: 0, +/// # idle_callbacks: 0, +/// # latency: LatencyStats::default(), +/// # error: self.stats.error(), +/// # } +/// # } +/// # } +/// +/// impl RunningCapture for Recording { +/// fn display_id(&self) -> u32 { +/// self.display.id +/// } +/// fn segment(&self) -> u32 { +/// self.segment +/// } +/// fn stats(&self) -> Arc { +/// self.stats.clone() +/// } +/// fn partial(&self) -> DisplayResult { +/// self.result(None, None) +/// } +/// fn stop(self: Box, reason: EndReason, end_mach_ns: Option) -> DisplayResult { +/// self.result(Some(reason), end_mach_ns) +/// } +/// } +/// +/// let display = DisplayDescriptor { +/// id: 2, +/// frame: [0.0, 0.0, 1920.0, 1080.0], +/// width_points: 1920, +/// height_points: 1080, +/// asleep: false, +/// scale: 1.0, +/// }; +/// let capture: Box = +/// Box::new(Recording { display, segment: 1, stats: Arc::default() }); +/// +/// assert_eq!(capture.last_signal_mach_ns(), None); +/// capture.keepalive(1_000_000_000); +/// +/// // A running segment has no end yet. +/// let so_far = capture.partial(); +/// assert_eq!((so_far.end_reason, so_far.video_end_mach_ns), (None, None)); +/// +/// // Stopping consumes the capture and returns the final description of the segment. +/// let result = capture.stop(EndReason::Stopped, None); +/// assert_eq!(result.end_reason, Some(EndReason::Stopped)); +/// assert_eq!(result.file, "video-2.mp4"); +/// ``` +pub trait RunningCapture { + /// Returns the id of the display being captured ([`DisplayDescriptor::id`]). + fn display_id(&self) -> u32; + /// Returns this capture's segment number ([`DisplayResult::segment`]). + fn segment(&self) -> u32; + /// Returns the live counters, shared with the capture's callbacks. + /// + /// The first error a capture reports goes here ([`CaptureStats::set_error`]), which is how + /// [`manager::DisplayManager`] learns that a stream died. + fn stats(&self) -> Arc; + /// Returns what is known about this segment so far, without ending it (for checkpoints). + /// + /// The result has no [`end_reason`](DisplayResult::end_reason) and no + /// [`video_end_mach_ns`](DisplayResult::video_end_mach_ns), and carries any error reported so + /// far. + fn partial(&self) -> DisplayResult; + /// Returns when this capture last showed a sign of life: any callback from the capture API, + /// even an idle one, or its start time if none has arrived yet. `None` if the backend cannot + /// tell, in which case the capture is never considered stalled. + /// + /// The macOS backend reports it, because `ScreenCaptureKit` keeps calling back about thirty + /// times a second on a healthy stream even when the screen is not changing. The Windows + /// backend returns `None`: Windows.Graphics.Capture sends no callbacks while the picture does + /// not change, so silence there is not a sign of death. The default is `None`. + fn last_signal_mach_ns(&self) -> Option { + None + } + /// Gives an idle capture the chance to keep itself moving. `now_ns` is the master-clock time + /// of the call. + /// + /// Called on every poll of [`manager::DisplayManager`] (the recorder polls several times a + /// second) for every live capture. A display whose screen is not changing produces no frames, + /// which would leave its file unflushed; this is where an idle capture keeps itself moving, by + /// repeating its last frame. The default does nothing. + fn keepalive(&self, _now_ns: u64) {} + /// Ends the segment, finalizes its file and returns the final description of it. + /// + /// `end_mach_ns` is when the segment really ended, if that is earlier than now (a stream that + /// died). Both backends clamp it to no earlier than the last frame written and no later than + /// the moment of the call, and end the file at that time, so that a display that stopped + /// changing does not leave a file that ends at its last change. + /// + /// This cannot fail. A problem stopping the capture or finalizing the file is recorded in + /// [`DisplayResult::error`], and a segment that never received a frame is marked there. It + /// blocks until the file is finalized. + fn stop(self: Box, reason: EndReason, end_mach_ns: Option) -> DisplayResult; +} + +/// Where displays and captures come from. +/// +/// The seam between the recording policy and the OS. [`manager::DisplayManager`] is generic over +/// it, so the policy is tested against a scripted backend and the OS code is the two methods below. +/// It is implemented by the macOS backend (`sck`), the Windows backend (`wgc`), [`NoDisplays`], +/// and [`Capture`], which forwards to one of the other two. [`crate::platform::capture::Backend`] +/// names the real one for the host. +/// +/// # The contract +/// +/// - [`active_displays`](CaptureBackend::active_displays) is polled several times a second, so it +/// must be cheap. +/// - An error from it means "could not ask", not "there are no displays". +/// [`manager::DisplayManager`] keeps what is running and reports it once. +/// - [`start`](CaptureBackend::start) is given segment numbers that are 1-based and per display. +/// A number is used up only by a capture that started: after a failed start, the next attempt +/// gets the same number. +/// - A started capture writes `video-[-].mp4` and `frames-[-].csv` into +/// the directory, named by [`video_file_name`] and [`frames_file_name`]. +/// +/// The compiler enforces that both methods are written: +/// +/// ```rust,compile_fail,E0046 +/// # use color_eyre::eyre; +/// # use komorebi_recorder::capture::{CaptureBackend, DisplayDescriptor}; +/// struct Screen; +/// +/// impl CaptureBackend for Screen { +/// fn active_displays(&self) -> eyre::Result> { +/// Ok(Vec::new()) +/// } +/// // no `start` +/// } +/// ``` +/// +/// # Example +/// +/// A machine with one display, whose capture does nothing. [`RunningCapture`] shows what the +/// capture itself implements; here it is out of sight. +/// +/// ```rust +/// use color_eyre::eyre; +/// use komorebi_recorder::capture::{ +/// CaptureBackend, CaptureConfig, DisplayDescriptor, RunningCapture, +/// }; +/// use std::path::Path; +/// # use komorebi_recorder::capture::{ +/// # CaptureStats, Codec, DisplayResult, EndReason, LatencyStats, frames_file_name, +/// # video_file_name, +/// # }; +/// # use std::sync::Arc; +/// # +/// # struct Recording { +/// # display: DisplayDescriptor, +/// # segment: u32, +/// # } +/// # +/// # impl Recording { +/// # fn result(&self, reason: Option, end: Option) -> DisplayResult { +/// # DisplayResult { +/// # display: self.display.clone(), +/// # segment: self.segment, +/// # scale: 1.0, +/// # width_px: 1920, +/// # height_px: 1080, +/// # codec: Codec::H264, +/// # file: video_file_name(self.display.id, self.segment), +/// # frames_file: frames_file_name(self.display.id, self.segment), +/// # video_t0_mach_ns: None, +/// # video_end_mach_ns: end, +/// # end_reason: reason, +/// # frames: 0, +/// # dropped: 0, +/// # idle_callbacks: 0, +/// # latency: LatencyStats::default(), +/// # error: None, +/// # } +/// # } +/// # } +/// # +/// # impl RunningCapture for Recording { +/// # fn display_id(&self) -> u32 { +/// # self.display.id +/// # } +/// # fn segment(&self) -> u32 { +/// # self.segment +/// # } +/// # fn stats(&self) -> Arc { +/// # Arc::default() +/// # } +/// # fn partial(&self) -> DisplayResult { +/// # self.result(None, None) +/// # } +/// # fn stop(self: Box, reason: EndReason, end: Option) -> DisplayResult { +/// # self.result(Some(reason), end) +/// # } +/// # } +/// +/// struct OneDisplay(DisplayDescriptor); +/// +/// impl CaptureBackend for OneDisplay { +/// fn active_displays(&self) -> eyre::Result> { +/// Ok(vec![self.0.clone()]) +/// } +/// +/// fn start( +/// &self, +/// display: &DisplayDescriptor, +/// segment: u32, +/// _config: &CaptureConfig, +/// _dir: &Path, +/// ) -> eyre::Result> { +/// if display.id != self.0.id { +/// eyre::bail!("display {} is not available for capture (yet)", display.id); +/// } +/// Ok(Box::new(Recording { display: display.clone(), segment })) +/// } +/// } +/// +/// let screen = DisplayDescriptor { +/// id: 2, +/// frame: [0.0, 0.0, 1920.0, 1080.0], +/// width_points: 1920, +/// height_points: 1080, +/// asleep: false, +/// scale: 1.0, +/// }; +/// let backend = OneDisplay(screen.clone()); +/// +/// let displays = backend.active_displays()?; +/// assert_eq!(displays, vec![screen.clone()]); +/// +/// let capture = backend.start(&displays[0], 1, &CaptureConfig::default(), Path::new("."))?; +/// assert_eq!((capture.display_id(), capture.segment()), (2, 1)); +/// +/// // A display that is not attached cannot be started. +/// let missing = DisplayDescriptor { id: 9, ..screen }; +/// assert!(backend.start(&missing, 1, &CaptureConfig::default(), Path::new(".")).is_err()); +/// # Ok::<(), Box>(()) +/// ``` +pub trait CaptureBackend { + /// Returns the displays that exist right now, asleep ones included (see + /// [`DisplayDescriptor::asleep`]). Must be cheap: it is polled several times a second. + /// + /// # Errors + /// + /// Returns an error when the OS cannot be asked for its displays. The caller treats that as + /// "unknown", not as "none": [`manager::DisplayManager::poll`] changes nothing and reports it + /// once. + fn active_displays(&self) -> eyre::Result>; + + /// Starts capturing `display` into `dir` as segment number `segment`, and returns the running + /// capture. + /// + /// # Errors + /// + /// Returns an error when the capture cannot begin: the display is not available for capture + /// (yet), the screen is locked, a permission is missing, or the file cannot be created. The + /// caller does not give up. [`manager::DisplayManager`] retries with a backoff and reports the + /// outage once. + fn start( + &self, + display: &DisplayDescriptor, + segment: u32, + config: &CaptureConfig, + dir: &Path, + ) -> eyre::Result>; +} + +/// The backend of a recording made without video: no display exists, so nothing is ever captured +/// and nothing about the screen is asked of the OS. +/// +/// [`active_displays`](CaptureBackend::active_displays) is always an empty list, and +/// [`start`](CaptureBackend::start) always fails. [`Capture::new`] builds it for +/// `record --no-video`. +/// +/// # Example +/// +/// ```rust +/// use komorebi_recorder::capture::{CaptureBackend, CaptureConfig, DisplayDescriptor, NoDisplays}; +/// use std::path::Path; +/// +/// assert!(NoDisplays.active_displays()?.is_empty()); +/// +/// let display = DisplayDescriptor { +/// id: 1, +/// frame: [0.0; 4], +/// width_points: 1, +/// height_points: 1, +/// asleep: false, +/// scale: 1.0, +/// }; +/// let error = NoDisplays +/// .start(&display, 1, &CaptureConfig::default(), Path::new(".")) +/// .err() +/// .expect("there is nothing to start"); +/// assert_eq!(error.to_string(), "this recording has no video"); +/// # Ok::<(), Box>(()) +/// ``` +pub struct NoDisplays; + +impl CaptureBackend for NoDisplays { + fn active_displays(&self) -> eyre::Result> { + Ok(Vec::new()) + } + + fn start( + &self, + _display: &DisplayDescriptor, + _segment: u32, + _config: &CaptureConfig, + _dir: &Path, + ) -> eyre::Result> { + Err(eyre::Report::msg("this recording has no video")) + } +} + +/// The host's capture, or none at all (`record --no-video`). +/// +/// This is the [`CaptureBackend`] the recorder runs [`manager::DisplayManager`] with. It forwards +/// every call to the backend it wraps, so the choice between recording video and not is made once, +/// when it is built, and nothing downstream branches on it. +pub enum Capture { + /// The backend of the host the recorder is running on: `ScreenCaptureKit` on macOS, + /// Windows.Graphics.Capture on Windows, and on any other host one whose methods fail with a + /// message that recording is not supported there. + Host(crate::platform::capture::Backend), + /// No video: see [`NoDisplays`]. + Nothing(NoDisplays), +} + +impl Capture { + /// Returns the host's backend if `video` is `true`, and [`NoDisplays`] otherwise. + /// + /// Building either asks the OS nothing: neither permission nor displays. + /// + /// # Example + /// + /// ```rust + /// use komorebi_recorder::capture::{Capture, CaptureBackend}; + /// + /// // A recording without video has no displays to follow, and no screen is asked about. + /// let capture = Capture::new(false); + /// assert!(matches!(capture, Capture::Nothing(_))); + /// assert!(capture.active_displays()?.is_empty()); + /// + /// // With video it is the host's backend. Listing its displays would ask the OS, so it is + /// // only built here. + /// assert!(matches!(Capture::new(true), Capture::Host(_))); + /// # Ok::<(), Box>(()) + /// ``` + #[must_use] + pub fn new(video: bool) -> Self { + if video { + Capture::Host(crate::platform::capture::Backend) + } else { + Capture::Nothing(NoDisplays) + } + } +} + +impl CaptureBackend for Capture { + fn active_displays(&self) -> eyre::Result> { + match self { + Capture::Host(backend) => backend.active_displays(), + Capture::Nothing(backend) => backend.active_displays(), + } + } + + fn start( + &self, + display: &DisplayDescriptor, + segment: u32, + config: &CaptureConfig, + dir: &Path, + ) -> eyre::Result> { + match self { + Capture::Host(backend) => backend.start(display, segment, config, dir), + Capture::Nothing(backend) => backend.start(display, segment, config, dir), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_recording_without_video_has_no_displays_to_follow() { + let backend = Capture::new(false); + assert!(backend.active_displays().expect("displays").is_empty()); + let display = DisplayDescriptor { + id: 1, + frame: [0.0; 4], + width_points: 1, + height_points: 1, + asleep: false, + scale: 1.0, + }; + assert!( + backend + .start(&display, 1, &CaptureConfig::default(), Path::new(".")) + .is_err() + ); + } + + #[test] + fn host_clock_check_accepts_measured_latencies_and_rejects_other_clocks() { + // measured on a real display + let measured = LatencyStats { + samples: 229, + min_ns: -7_506_167, + max_ns: 196_015_958, + mean_ns: 6_524_913, + }; + assert!(measured.looks_like_host_clock()); + + // a different clock (e.g. wall time vs mach time) is off by far more than this + let wrong = LatencyStats { + samples: 10, + min_ns: 3_600_000_000_000, + max_ns: 3_600_100_000_000, + mean_ns: 3_600_050_000_000, + }; + assert!(!wrong.looks_like_host_clock()); + + let negative = LatencyStats { + samples: 10, + min_ns: -3_000_000_000, + max_ns: -2_900_000_000, + mean_ns: -2_950_000_000, + }; + assert!(!negative.looks_like_host_clock()); + + assert!( + LatencyStats::default().looks_like_host_clock(), + "no frames, nothing to doubt" + ); + } + + #[test] + fn segment_file_names_keep_the_first_segment_plain() { + assert_eq!(video_file_name(2, 1), "video-2.mp4"); + assert_eq!(frames_file_name(2, 1), "frames-2.csv"); + assert_eq!(video_file_name(2, 2), "video-2-2.mp4"); + assert_eq!(frames_file_name(2, 3), "frames-2-3.csv"); + // a display id that ends in the same digits as a segment number cannot collide + assert_ne!(video_file_name(1, 2), video_file_name(12, 1)); + } + + #[test] + fn only_a_size_change_invalidates_a_capture() { + let display = |x: f64, w: i64, h: i64| DisplayDescriptor { + id: 1, + frame: [x, 0.0, w as f64, h as f64], + width_points: w, + height_points: h, + asleep: false, + scale: 1.0, + }; + + assert!(!display(0.0, 1800, 1169).size_differs(&display(-1800.0, 1800, 1169))); + assert!(display(0.0, 1800, 1169).size_differs(&display(0.0, 1512, 982))); + } + + #[test] + fn only_the_first_error_is_kept() { + let stats = CaptureStats::default(); + assert_eq!(stats.error(), None); + stats.set_error("first".into(), 100); + stats.set_error("second".into(), 200); + assert_eq!(stats.error().as_deref(), Some("first")); + assert_eq!(stats.error_at_mach_ns(), Some(100)); + } + + #[test] + fn old_manifests_without_segment_fields_still_parse() { + let old = r#"{ + "display": {"id": 2, "frame": [0.0, 0.0, 3840.0, 1620.0], "width_points": 3840, "height_points": 1620}, + "scale": 1.0, "width_px": 3840, "height_px": 1620, "codec": "hevc", + "file": "video-2.mp4", "frames_file": "frames-2.csv", + "video_t0_mach_ns": 5, "frames": 3, "dropped": 0, + "latency": {"samples": 0, "min_ns": 0, "max_ns": 0, "mean_ns": 0}, + "error": null + }"#; + let result: DisplayResult = serde_json::from_str(old).expect("old format parses"); + assert_eq!(result.segment, 1); + assert_eq!(result.video_end_mach_ns, None); + assert_eq!(result.end_reason, None); + } +} diff --git a/komorebi-recorder/src/capture/sck.rs b/komorebi-recorder/src/capture/sck.rs new file mode 100644 index 00000000..1380c7ed --- /dev/null +++ b/komorebi-recorder/src/capture/sck.rs @@ -0,0 +1,1128 @@ +//! The macOS capture backend: `ScreenCaptureKit` frames written straight to an `AVAssetWriter` MP4. +//! +//! This module exists on macOS only. [`SckBackend`] is the [`CaptureBackend`] and +//! [`DisplayCapture`] the [`RunningCapture`]; [`crate::platform::capture`] re-exports +//! `SckBackend` as the host's `Backend`, together with [`active_displays`], [`ensure_permission`], +//! [`has_permission`] and [`media_duration_s`]. The Objective-C interop for capturing is isolated +//! here (decoding recorded video is `av`). +//! +//! A frame's `CMSampleBuffer` presentation timestamp is on the host clock (the same clock as +//! `mach_absolute_time`), which is what lets the video line up exactly with the unified log. It is +//! converted to nanoseconds on the master clock with the host timebase, and the difference between +//! it and the time the callback ran is recorded in the segment's +//! [`LatencyStats`](crate::capture::LatencyStats). +//! +//! # How a segment is recorded +//! +//! 1. [`DisplayCapture::start`] finds the display among `ScreenCaptureKit`'s shareable content, +//! builds a filter that leaves out this process and any other `komorebi-recorder` process, and +//! configures the stream: the size, `fps` as the minimum frame interval, BGRA pixels, the cursor +//! shown. It creates the `AVAssetWriter` (an MP4 cut into movie fragments every second, so a +//! killed recorder leaves a file that plays up to the last fragment) and the frame log with its +//! header row, and starts the stream. +//! 2. Each frame arrives on a serial dispatch queue. A callback that carries no new content (the +//! frame status is idle, blank or suspended) is counted in +//! [`CaptureStats::idle_callbacks`] and dropped. A real frame starts the writer's session at its +//! timestamp if it is the first, and is appended and logged. If the writer is not ready for more +//! media data the frame is dropped and counted in [`CaptureStats::dropped`]. A frame that is not +//! later than the last thing appended is moved 1 ms after it. +//! 3. [`RunningCapture::keepalive`] re-submits the last frame when the display has shown nothing +//! new for half a second and the stream has called back within the last two seconds. A static +//! screen advances neither media time nor the fragments, so without this a crash would lose +//! everything since the last change. +//! 4. [`RunningCapture::stop`] waits up to three seconds for `ScreenCaptureKit` to confirm that the +//! stream stopped (not at all when it had already reported an error), lets a frame callback in +//! flight finish, ends the writer's session at the segment's end time and waits up to ten +//! seconds for the file to be finalized. A timeout is reported in +//! [`DisplayResult::error`]. +//! +//! The placement, frame-log and keep-alive rules of steps 2 and 3 are the ones in +//! [`crate::capture::segment`], written inline here with the same numbers. This module uses only +//! [`even`] and [`Latency`] from there. +//! +//! # Permission +//! +//! Capturing needs Screen Recording permission. [`has_permission`] asks without prompting and +//! [`ensure_permission`] prompts. The grant attaches to the responsible process (normally the +//! terminal the binary was launched from), exactly as it does for komorebi itself. +//! +//! # Tests +//! +//! The only unit test here is for [`is_recorder_application`]. Capturing needs a screen and the +//! permission, so [`DisplayCapture`] is exercised by `doctor --probe capture` and by recordings; +//! the policy above it is tested against a scripted backend in [`crate::capture::manager`]. + +use crate::capture::CaptureBackend; +use crate::capture::CaptureConfig; +use crate::capture::CaptureStats; +use crate::capture::Codec; +use crate::capture::DisplayDescriptor; +use crate::capture::DisplayResult; +use crate::capture::EndReason; +use crate::capture::RunningCapture; +use crate::capture::frames_file_name; +use crate::capture::segment::Latency; +use crate::capture::segment::even; +use crate::capture::video_file_name; +use crate::clock; +use block2::RcBlock; +use color_eyre::eyre; +use color_eyre::eyre::WrapErr; +use dispatch2::DispatchQueue; +use dispatch2::DispatchQueueAttr; +use dispatch2::DispatchRetained; +use objc2::AllocAnyThread; +use objc2::DefinedClass; +use objc2::define_class; +use objc2::msg_send; +use objc2::rc::Retained; +use objc2::runtime::AnyObject; +use objc2::runtime::NSObject; +use objc2::runtime::NSObjectProtocol; +use objc2::runtime::ProtocolObject; +use objc2_av_foundation::AVAsset; +use objc2_av_foundation::AVAssetWriter; +use objc2_av_foundation::AVAssetWriterInput; +use objc2_av_foundation::AVFileTypeMPEG4; +use objc2_av_foundation::AVMediaTypeVideo; +use objc2_av_foundation::AVVideoCodecKey; +use objc2_av_foundation::AVVideoCodecTypeH264; +use objc2_av_foundation::AVVideoCodecTypeHEVC; +use objc2_av_foundation::AVVideoHeightKey; +use objc2_av_foundation::AVVideoWidthKey; +use objc2_core_foundation::CFDictionary; +use objc2_core_foundation::CFRetained; +use objc2_core_graphics::CGDisplayBounds; +use objc2_core_graphics::CGDisplayCopyDisplayMode; +use objc2_core_graphics::CGDisplayIsAsleep; +use objc2_core_graphics::CGDisplayMode; +use objc2_core_graphics::CGError; +use objc2_core_graphics::CGGetActiveDisplayList; +use objc2_core_graphics::CGMainDisplayID; +use objc2_core_graphics::CGPreflightScreenCaptureAccess; +use objc2_core_graphics::CGRequestScreenCaptureAccess; +use objc2_core_media::CMClock; +use objc2_core_media::CMSampleBuffer; +use objc2_core_media::CMSampleTimingInfo; +use objc2_core_media::CMTime; +use objc2_core_media::kCMTimeInvalid; +use objc2_foundation::NSArray; +use objc2_foundation::NSDictionary; +use objc2_foundation::NSError; +use objc2_foundation::NSNumber; +use objc2_foundation::NSString; +use objc2_foundation::NSURL; +use objc2_screen_capture_kit::SCContentFilter; +use objc2_screen_capture_kit::SCFrameStatus; +use objc2_screen_capture_kit::SCRunningApplication; +use objc2_screen_capture_kit::SCShareableContent; +use objc2_screen_capture_kit::SCStream; +use objc2_screen_capture_kit::SCStreamConfiguration; +use objc2_screen_capture_kit::SCStreamDelegate; +use objc2_screen_capture_kit::SCStreamFrameInfoStatus; +use objc2_screen_capture_kit::SCStreamOutput; +use objc2_screen_capture_kit::SCStreamOutputType; +use objc2_screen_capture_kit::SCWindow; +use parking_lot::Mutex; +use std::ffi::c_void; +use std::fs::File; +use std::io::BufWriter; +use std::io::Write; +use std::path::Path; +use std::ptr::NonNull; +use std::sync::Arc; +use std::sync::atomic::Ordering; +use std::sync::mpsc; +use std::time::Duration; +use std::time::Instant; + +/// `kCVPixelFormatType_32BGRA` +const PIXEL_FORMAT_BGRA: u32 = 0x4247_5241; + +const CALLBACK_TIMEOUT: Duration = Duration::from_secs(10); + +/// How often a movie fragment is flushed to disk, which bounds how much video a crash can lose. +const FRAGMENT_SECONDS: i64 = 1; + +/// How long a display may go without a new frame before its last frame is re-submitted. +/// +/// `ScreenCaptureKit` only produces frames when the screen *changes*, and movie fragments are cut +/// on media time. A static screen therefore advances neither: its fragments are never flushed, so +/// a crash would lose everything since the last change, and a display that never changed after +/// the first second would leave an unreadable file. Re-submitting the last frame costs a few +/// bytes (it is identical) and keeps the encoder and the file moving. +const KEEPALIVE_AFTER: Duration = Duration::from_millis(500); + +/// A stream silent for longer than this is not kept alive (see `keepalive`). +const KEEPALIVE_STREAM_TIMEOUT_NS: u64 = 2_000_000_000; + +/// How often the frame timestamp log is flushed, which bounds what a crash can lose from it. +const CSV_FLUSH_INTERVAL: Duration = Duration::from_secs(1); + +/// Stopping a stream that is already dead has nothing to wait for; a live one answers quickly. +const STOP_TIMEOUT: Duration = Duration::from_secs(3); + +/// Returns `true` if and only if Screen Recording permission is already granted. Never prompts. +#[must_use] +pub fn has_permission() -> bool { + CGPreflightScreenCaptureAccess() +} + +/// Makes sure this process has Screen Recording permission, prompting the user if it does not. +/// +/// The grant attaches to the *responsible* process (normally the terminal the binary was +/// launched from), exactly as it does for komorebi itself. +/// +/// # Errors +/// +/// Returns an error when the permission is not granted, whether already or after the prompt this +/// call showed. The message names where to grant it (System Settings > Privacy & Security > +/// Screen & System Audio Recording) and tells the user to restart the recorder afterwards. +pub fn ensure_permission() -> eyre::Result<()> { + // connect to the window server; without this ScreenCaptureKit asserts in a bare CLI process + let _ = CGMainDisplayID(); + + if CGPreflightScreenCaptureAccess() { + return Ok(()); + } + + if CGRequestScreenCaptureAccess() { + return Ok(()); + } + + Err(eyre::Report::msg( + "Screen Recording permission is required: grant it to the terminal (or app) launching \ + komorebi-recorder in System Settings > Privacy & Security > Screen & System Audio \ + Recording, then restart it", + )) +} + +fn error_message(error: *mut NSError) -> Option { + // SAFETY: ScreenCaptureKit passes either null or a valid NSError for the duration of the + // completion handler + unsafe { error.as_ref() }.map(|error| error.localizedDescription().to_string()) +} + +/// Run an asynchronous `ScreenCaptureKit` call that reports through an `NSError` completion +/// handler and wait for it. +fn wait_for_completion( + what: &str, + timeout: Duration, + call: impl FnOnce(&RcBlock), +) -> eyre::Result<()> { + let (tx, rx) = mpsc::channel(); + let block = RcBlock::new(move |error: *mut NSError| { + let _ = tx.send(error_message(error)); + }); + + call(&block); + + match rx.recv_timeout(timeout) { + Ok(None) => Ok(()), + Ok(Some(message)) => Err(eyre::Report::msg(format!("{what} failed: {message}"))), + Err(_) => Err(eyre::Report::msg(format!("{what} timed out"))), + } +} + +fn shareable_content() -> eyre::Result> { + ensure_permission()?; + + let (tx, rx) = mpsc::channel(); + let block = RcBlock::new( + move |content: *mut SCShareableContent, error: *mut NSError| { + // SAFETY: a non-null content pointer is a valid +0 object that we retain before the + // handler returns + let content = unsafe { Retained::retain(content) }; + let _ = tx.send((content.map(SendRetained), error_message(error))); + }, + ); + + // SAFETY: the block outlives the call and matches the expected signature + unsafe { SCShareableContent::getShareableContentWithCompletionHandler(&block) }; + + match rx.recv_timeout(CALLBACK_TIMEOUT) { + Ok((Some(content), _)) => Ok(content.0), + Ok((None, error)) => Err(eyre::Report::msg(format!( + "could not enumerate displays: {}", + error.unwrap_or_else(|| "unknown error".to_string()) + ))), + Err(_) => Err(eyre::Report::msg("timed out enumerating displays")), + } +} + +/// An Objective-C object we promise is safe to hand across threads. Only used to move the result +/// of a completion handler to the waiting thread. +struct SendRetained(Retained); + +// SAFETY: the wrapped object is only used by the thread that receives it +unsafe impl Send for SendRetained {} + +/// Returns the playable duration of a video file as `AVFoundation` reads it, in seconds. +/// +/// This is what recovery trusts for a file whose recorder died: the frame log is flushed on a +/// timer and can lag behind (or run ahead of) what actually reached the video. `ffprobe`'s +/// duration for a truncated fragmented MP4 under-reads, so it is not used for this. +/// +/// Returns `None` if the path is not valid UTF-8, or `AVFoundation` reports no usable duration for +/// the file (a negative one, or one with no timescale). +#[must_use] +pub fn media_duration_s(path: &Path) -> Option { + let path = path.to_str()?; + // SAFETY: plain object construction and a property read on a live object + let duration = unsafe { + let url = NSURL::fileURLWithPath(&NSString::from_str(path)); + AVAsset::assetWithURL(&url).duration() + }; + + (duration.timescale > 0 && duration.value >= 0) + .then(|| duration.value as f64 / f64::from(duration.timescale)) +} + +/// Returns `true` if and only if a running application is this recorder or another +/// `komorebi-recorder` process, which should be left out of the capture. +/// +/// An application matches when its `pid` is `own_pid`, or when its `name` is `komorebi-recorder` +/// (compared without regard to ASCII case) as a whole: the viewer is a separate process, often open +/// beside a recording, and must stay out of the video too. Names that only start with it, and +/// komorebi itself, do not match. +/// +/// # Example +/// +/// ```rust +/// use komorebi_recorder::capture::sck::is_recorder_application; +/// +/// // This process, by pid. +/// assert!(is_recorder_application(42, "whatever", 42)); +/// // Another recorder, by name. +/// assert!(is_recorder_application(7, "Komorebi-Recorder", 42)); +/// // Other applications, including komorebi itself and similarly named ones. +/// assert!(!is_recorder_application(7, "komorebi", 42)); +/// assert!(!is_recorder_application(7, "komorebi-recorder-helper", 42)); +/// ``` +#[must_use] +pub fn is_recorder_application(pid: i32, name: &str, own_pid: i32) -> bool { + pid == own_pid || name.eq_ignore_ascii_case("komorebi-recorder") +} + +/// How many pixels the display really has for each point: the display mode's pixel width over its +/// width in points (`CGDisplayPixelsWide` would not do, it reports points for a scaled mode). +fn backing_scale(id: u32) -> f64 { + let Some(mode) = CGDisplayCopyDisplayMode(id) else { + return 1.0; + }; + + let (pixels, points) = ( + CGDisplayMode::pixel_width(Some(&mode)), + CGDisplayMode::width(Some(&mode)), + ); + if points > 0 && pixels > 0 { + pixels as f64 / points as f64 + } else { + 1.0 + } +} + +/// Returns the displays that are attached and drawable right now, from CoreGraphics. +/// +/// This is deliberately not `ScreenCaptureKit`: it is synchronous and cheap enough to poll several +/// times a second, needs no permission, and is what tells us a display was unplugged or has gone +/// to sleep before any stream notices. +/// +/// Each descriptor has its bounds in global logical points, whether the display is asleep, and its +/// backing scale (see [`DisplayDescriptor::scale`]). At most 32 displays are reported. +/// +/// # Errors +/// +/// Returns an error when `CGGetActiveDisplayList` fails; the message has its status code. +pub fn active_displays() -> eyre::Result> { + // a connection to the window server is required before any display call + let _ = CGMainDisplayID(); + + let mut ids = [0_u32; 32]; + let mut count = 0_u32; + // SAFETY: both pointers are valid for the duration of the call and `ids` has room for + // `ids.len()` entries + let status = + unsafe { CGGetActiveDisplayList(ids.len() as u32, ids.as_mut_ptr(), &raw mut count) }; + if status != CGError::Success { + eyre::bail!("CGGetActiveDisplayList failed with {}", status.0); + } + + Ok(ids[..(count as usize).min(ids.len())] + .iter() + .map(|id| { + let bounds = CGDisplayBounds(*id); + DisplayDescriptor { + id: *id, + frame: [ + bounds.origin.x, + bounds.origin.y, + bounds.size.width, + bounds.size.height, + ], + width_points: bounds.size.width.round() as i64, + height_points: bounds.size.height.round() as i64, + asleep: CGDisplayIsAsleep(*id), + scale: backing_scale(*id), + } + }) + .collect()) +} + +/// The real, `ScreenCaptureKit`-backed [`CaptureBackend`]. +/// +/// [`active_displays`](CaptureBackend::active_displays) is [`active_displays`], and +/// [`start`](CaptureBackend::start) starts a [`DisplayCapture`]. On a Mac, +/// [`crate::platform::capture::Backend`] is this type. +pub struct SckBackend; + +impl CaptureBackend for SckBackend { + fn active_displays(&self) -> eyre::Result> { + active_displays() + } + + fn start( + &self, + display: &DisplayDescriptor, + segment: u32, + config: &CaptureConfig, + dir: &Path, + ) -> eyre::Result> { + Ok(Box::new(DisplayCapture::start( + display, segment, config, dir, + )?)) + } +} + +/// Everything the frame callback mutates, guarded by a mutex so `stop` can finalize safely. +struct WriterState { + writer: Retained, + input: Retained, + frames_csv: BufWriter, + last_flush: Instant, + started: bool, + finished: bool, + frame_index: u64, + latency: Latency, + /// The most recent real frame, kept so an idle display can be kept alive. + last_sample: Option>, + /// Presentation time (mach ns) of the last sample appended, real or keep-alive. Everything + /// appended must be strictly later than this. + last_append_ns: u64, +} + +struct Shared { + stats: Arc, + state: Mutex, +} + +// SAFETY: the Objective-C objects inside `WriterState` are only ever touched while holding the +// mutex, and `AVAssetWriter`/`AVAssetWriterInput` have no thread affinity +unsafe impl Send for Shared {} +// SAFETY: see above +unsafe impl Sync for Shared {} + +/// The `SCStreamOutput` frame sink and `SCStreamDelegate` error sink. +struct OutputIvars { + shared: Arc, +} + +define_class!( + // SAFETY: NSObject has no subclassing requirements and this class does not implement Drop + #[unsafe(super(NSObject))] + #[name = "KomorebiRecorderStreamOutput"] + #[ivars = OutputIvars] + struct StreamOutput; + + unsafe impl NSObjectProtocol for StreamOutput {} + + unsafe impl SCStreamOutput for StreamOutput { + #[unsafe(method(stream:didOutputSampleBuffer:ofType:))] + fn stream_did_output_sample_buffer_of_type( + &self, + _stream: &SCStream, + sample_buffer: &CMSampleBuffer, + kind: SCStreamOutputType, + ) { + if kind == SCStreamOutputType::Screen { + handle_frame(&self.ivars().shared, sample_buffer); + } + } + } + + unsafe impl SCStreamDelegate for StreamOutput { + #[unsafe(method(stream:didStopWithError:))] + fn stream_did_stop_with_error(&self, _stream: &SCStream, error: &NSError) { + self.ivars().shared.stats.set_error( + format!("stream stopped: {}", error.localizedDescription()), + clock::mono_now_ns(), + ); + } + } +); + +impl StreamOutput { + fn new(shared: Arc) -> Retained { + let this = Self::alloc().set_ivars(OutputIvars { shared }); + // SAFETY: NSObject's `init` is always valid + unsafe { msg_send![super(this), init] } + } +} + +/// Read the `SCFrameStatus` attached to a sample buffer, if present. +/// +/// # Safety +/// +/// `sample` must be a valid sample buffer delivered by `ScreenCaptureKit`. +unsafe fn frame_status(sample: &CMSampleBuffer) -> Option { + let attachments = unsafe { sample.sample_attachments_array(false) }?; + if attachments.count() == 0 { + return None; + } + + let dictionary = unsafe { attachments.value_at_index(0) }.cast::(); + // SAFETY: element 0 of the attachments array is a CFDictionary + let dictionary = unsafe { dictionary.as_ref() }?; + let key = std::ptr::from_ref::(unsafe { SCStreamFrameInfoStatus }).cast::(); + let value = unsafe { dictionary.value(key) }.cast::(); + // SAFETY: the status value is an NSNumber when present + unsafe { value.as_ref() }.map(NSNumber::integerValue) +} + +fn handle_frame(shared: &Shared, sample: &CMSampleBuffer) { + let received_ns = clock::mono_now_ns(); + shared.stats.callbacks.fetch_add(1, Ordering::Relaxed); + shared + .stats + .last_callback_mach_ns + .store(received_ns, Ordering::Relaxed); + + // SAFETY: `sample` is a live sample buffer for the duration of the callback + let (pts, pts_ns) = unsafe { + if !sample.is_valid() || !sample.data_is_ready() { + return; + } + + // idle/blank/suspended frames carry no new content + if let Some(status) = frame_status(sample) + && status != SCFrameStatus::Complete.0 + { + shared.stats.idle_callbacks.fetch_add(1, Ordering::Relaxed); + return; + } + + if sample.image_buffer().is_none() { + return; + } + + let pts = sample.presentation_time_stamp(); + let ticks = CMClock::convert_host_time_to_system_units(pts); + (pts, clock::timebase().ticks_to_ns(ticks)) + }; + + let mut state = shared.state.lock(); + if state.finished { + return; + } + + if !state.started { + // SAFETY: the writer is configured and has an input attached + let started = unsafe { + if state.writer.startWriting() { + state.writer.startSessionAtSourceTime(pts); + true + } else { + false + } + }; + + if !started { + shared.stats.set_error( + format!("could not start writing: {}", writer_error(&state)), + received_ns, + ); + state.finished = true; + return; + } + + state.started = true; + shared + .stats + .first_pts_mach_ns + .store(pts_ns, Ordering::Relaxed); + } + + // SAFETY: the input belongs to a writer that is currently writing + let ready = unsafe { state.input.isReadyForMoreMediaData() }; + if !ready { + shared.stats.dropped.fetch_add(1, Ordering::Relaxed); + return; + } + + // A keep-alive may have been stamped a hair ahead of this real frame (frame timestamps run + // slightly ahead of when they are received). The writer rejects out-of-order samples, so move + // the frame just past it rather than lose the frame or the segment. + let mut pts_ns = pts_ns; + let adjusted = if pts_ns <= state.last_append_ns { + pts_ns = state.last_append_ns + 1_000_000; + retimed(sample, pts_ns) + } else { + None + }; + + // SAFETY: as above + let appended = unsafe { + match &adjusted { + Some(copy) => state.input.appendSampleBuffer(copy), + None => state.input.appendSampleBuffer(sample), + } + }; + + if !appended { + shared.stats.set_error( + format!("could not append frame: {}", writer_error(&state)), + received_ns, + ); + state.finished = true; + return; + } + + state.last_append_ns = pts_ns; + // SAFETY: `sample` is a valid sample buffer; retaining it keeps it alive past the callback + state.last_sample = Some(unsafe { CFRetained::retain(NonNull::from(sample)) }); + + let index = state.frame_index; + state.frame_index += 1; + state.latency.record(received_ns as i64 - pts_ns as i64); + + // a failed csv write must not take down the capture; the mp4 is the source of truth + let _ = writeln!(state.frames_csv, "{index},{pts_ns},{received_ns}"); + if state.last_flush.elapsed() >= CSV_FLUSH_INTERVAL { + // flushed on whole lines, so a crash never leaves a torn row behind + let _ = state.frames_csv.flush(); + state.last_flush = Instant::now(); + } + + shared.stats.frames.fetch_add(1, Ordering::Relaxed); + shared + .stats + .last_pts_mach_ns + .store(pts_ns, Ordering::Relaxed); +} + +/// A copy of `sample` presented at `pts_ns` on the host clock. +fn retimed(sample: &CMSampleBuffer, pts_ns: u64) -> Option> { + let ticks = clock::timebase().ns_to_ticks(pts_ns); + // SAFETY: plain value construction and a Core Media call whose out pointer is valid; on + // success the returned buffer is +1 and ownership is taken by `CFRetained::from_raw` + unsafe { + let timing = CMSampleTimingInfo { + duration: kCMTimeInvalid, + presentationTimeStamp: CMClock::make_host_time_from_system_units(ticks), + decodeTimeStamp: kCMTimeInvalid, + }; + let mut out: *mut CMSampleBuffer = std::ptr::null_mut(); + let status = CMSampleBuffer::create_copy_with_new_timing( + None, + sample, + 1, + &raw const timing, + NonNull::from(&mut out), + ); + + if status != 0 { + return None; + } + + NonNull::new(out).map(|out| CFRetained::from_raw(out)) + } +} + +fn writer_error(state: &WriterState) -> String { + // SAFETY: plain property reads on a live object + unsafe { + state.writer.error().map_or_else( + || "unknown error".to_string(), + |e| e.localizedDescription().to_string(), + ) + } +} + +/// An in-progress capture of one display: one segment, recorded by `ScreenCaptureKit` into an +/// `AVAssetWriter`. +/// +/// Built by [`DisplayCapture::start`] and used through [`RunningCapture`]. Its liveness signal is +/// the time of the last callback of any kind, idle ones included, or its start time before the +/// first arrives, so [`DisplayManager`](crate::capture::manager::DisplayManager) can tell a stalled +/// stream. [`stop`](RunningCapture::stop) always returns a [`DisplayResult`]: a segment that never +/// received a frame has the error `no frames were captured`. +pub struct DisplayCapture { + descriptor: DisplayDescriptor, + segment: u32, + started_mach_ns: u64, + scale: f64, + width_px: usize, + height_px: usize, + codec: Codec, + file: String, + frames_file: String, + stream: Retained, + // kept alive for as long as the stream may call into it + _output: Retained, + queue: DispatchRetained, + shared: Arc, +} + +impl DisplayCapture { + /// Starts capturing `display` into `dir` as the given segment, excluding this process's own + /// windows. + /// + /// Returns once `ScreenCaptureKit` confirms the stream started, without waiting for a frame: + /// `video_t0_mach_ns` is `None` until one arrives. The video is `config.codec` at the + /// display's size in points, or at its native pixels when `config.retina` is set, rounded to + /// even numbers. The files are `video-[-].mp4` and `frames-[-].csv` + /// in `dir`. + /// + /// # Errors + /// + /// Returns an error when: + /// + /// - Screen Recording permission is missing ([`ensure_permission`]); + /// - `ScreenCaptureKit` cannot list its shareable content, or takes more than ten seconds to + /// (`could not enumerate displays`); + /// - `display` is not among that content, which is the usual case for a display that has only + /// just appeared or is not ready (`display is not available for capture (yet)`); + /// - the video writer cannot be created: `dir` is not valid UTF-8, `AVFoundation` refuses the + /// file or the codec, or the writer rejects the input configuration; + /// - the frame log cannot be created in `dir`; + /// - the frame output cannot be attached to the stream; + /// - the stream does not start, or does not within ten seconds. The two files created for the + /// segment are removed in this case. + /// + /// A caller that retries gets the same segment number again: see [`CaptureBackend::start`]. + pub fn start( + display: &DisplayDescriptor, + segment: u32, + config: &CaptureConfig, + dir: &Path, + ) -> eyre::Result { + let content = shareable_content()?; + + // SAFETY: plain property reads on live objects + let (sc_display, own_apps) = unsafe { + let displays = content.displays(); + let sc_display = displays + .iter() + .find(|candidate| candidate.displayID() == display.id) + .ok_or_else(|| { + eyre::Report::msg(format!( + "display {} is not available for capture (yet)", + display.id + )) + })?; + + // this process, and any other komorebi-recorder (the viewer is its own process and is + // often open beside a recording): neither belongs in the video + let own_pid = std::process::id() as i32; + let own_apps: Vec> = content + .applications() + .iter() + .filter(|app| { + is_recorder_application( + app.processID(), + &app.applicationName().to_string(), + own_pid, + ) + }) + .collect(); + + (sc_display, own_apps) + }; + + // SAFETY: the filter is built from live objects, and the arrays outlive the call + let filter = unsafe { + let excluded = NSArray::from_retained_slice(&own_apps); + let no_windows: Retained> = NSArray::new(); + SCContentFilter::initWithDisplay_excludingApplications_exceptingWindows( + SCContentFilter::alloc(), + &sc_display, + &excluded, + &no_windows, + ) + }; + + // SAFETY: plain property read on a live object + let scale = if config.retina { + f64::from(unsafe { SCShareableContent::infoForFilter(&filter).pointPixelScale() }) + } else { + 1.0 + }; + + let width_px = even(display.width_points as f64 * scale); + let height_px = even(display.height_points as f64 * scale); + + let stream_config = unsafe { + let stream_config = SCStreamConfiguration::new(); + stream_config.setWidth(width_px); + stream_config.setHeight(height_px); + stream_config.setMinimumFrameInterval(CMTime::new(1, config.fps.max(1) as i32)); + stream_config.setPixelFormat(PIXEL_FORMAT_BGRA); + stream_config.setShowsCursor(true); + stream_config.setQueueDepth(6); + stream_config + }; + + let file = video_file_name(display.id, segment); + let frames_file = frames_file_name(display.id, segment); + let writer = create_writer(&dir.join(&file), config.codec, width_px, height_px)?; + let mut frames_csv = BufWriter::new( + File::create(dir.join(&frames_file)) + .wrap_err_with(|| format!("could not create {frames_file}"))?, + ); + writeln!(frames_csv, "index,pts_mach_ns,received_mach_ns")?; + frames_csv.flush()?; + + let stats = Arc::new(CaptureStats::default()); + let shared = Arc::new(Shared { + stats, + state: Mutex::new(WriterState { + writer: writer.0, + input: writer.1, + frames_csv, + last_flush: Instant::now(), + started: false, + finished: false, + frame_index: 0, + latency: Latency::default(), + last_sample: None, + last_append_ns: 0, + }), + }); + + let output = StreamOutput::new(shared.clone()); + let queue = DispatchQueue::new("komorebi-recorder.frames", DispatchQueueAttr::SERIAL); + + // SAFETY: all arguments are live objects; the output and queue are retained by `Self` + let stream = unsafe { + let stream = SCStream::initWithFilter_configuration_delegate( + SCStream::alloc(), + &filter, + &stream_config, + Some(ProtocolObject::from_ref(&*output)), + ); + + stream + .addStreamOutput_type_sampleHandlerQueue_error( + ProtocolObject::from_ref(&*output), + SCStreamOutputType::Screen, + Some(&queue), + ) + .map_err(|error| { + eyre::Report::msg(format!( + "could not attach the frame output: {}", + error.localizedDescription() + )) + })?; + + stream + }; + + // SAFETY: the stream is fully configured + let started = wait_for_completion( + "starting the screen capture", + CALLBACK_TIMEOUT, + |block| unsafe { + stream.startCaptureWithCompletionHandler(Some(block)); + }, + ); + + if let Err(error) = started { + // nothing was captured, so leave no empty segment files behind + let mut state = shared.state.lock(); + state.finished = true; + // SAFETY: the writer never started + unsafe { state.writer.cancelWriting() }; + drop(state); + let _ = std::fs::remove_file(dir.join(&file)); + let _ = std::fs::remove_file(dir.join(&frames_file)); + return Err(error); + } + + Ok(Self { + descriptor: display.clone(), + segment, + started_mach_ns: clock::mono_now_ns(), + scale, + width_px, + height_px, + codec: config.codec, + file, + frames_file, + stream, + _output: output, + queue, + shared, + }) + } + + fn result( + &self, + reason: Option, + end_mach_ns: Option, + error: Option, + ) -> DisplayResult { + let state = self.shared.state.lock(); + let stats = &self.shared.stats; + + DisplayResult { + display: self.descriptor.clone(), + segment: self.segment, + scale: self.scale, + width_px: self.width_px, + height_px: self.height_px, + codec: self.codec, + file: self.file.clone(), + frames_file: self.frames_file.clone(), + video_t0_mach_ns: stats.first_pts_mach_ns(), + video_end_mach_ns: end_mach_ns, + end_reason: reason, + frames: state.frame_index, + dropped: stats.dropped.load(Ordering::Relaxed), + idle_callbacks: stats.idle_callbacks.load(Ordering::Relaxed), + latency: state.latency.finish(), + error, + } + } +} + +impl RunningCapture for DisplayCapture { + fn display_id(&self) -> u32 { + self.descriptor.id + } + + fn segment(&self) -> u32 { + self.segment + } + + fn stats(&self) -> Arc { + self.shared.stats.clone() + } + + fn partial(&self) -> DisplayResult { + self.result(None, None, self.shared.stats.error()) + } + + fn last_signal_mach_ns(&self) -> Option { + Some( + self.shared + .stats + .last_callback_mach_ns() + .unwrap_or(self.started_mach_ns), + ) + } + + fn keepalive(&self, now_ns: u64) { + // Only keep a display alive while its stream is demonstrably alive. Re-submitting the last + // frame of a stream that has silently died would produce a frozen video that looks + // healthy while the screen moves on. + let alive = self + .shared + .stats + .last_callback_mach_ns() + .is_some_and(|last| now_ns.saturating_sub(last) < KEEPALIVE_STREAM_TIMEOUT_NS); + if !alive { + return; + } + + let mut state = self.shared.state.lock(); + if !state.started || state.finished { + return; + } + + let idle_ns = now_ns.saturating_sub(state.last_append_ns); + if idle_ns < u64::try_from(KEEPALIVE_AFTER.as_nanos()).unwrap_or(u64::MAX) { + return; + } + + let Some(last) = state.last_sample.clone() else { + return; + }; + + let pts_ns = now_ns.max(state.last_append_ns + 1_000_000); + let Some(copy) = retimed(&last, pts_ns) else { + return; + }; + + // SAFETY: the input belongs to a writer that is currently writing + if !unsafe { state.input.isReadyForMoreMediaData() } { + return; + } + + // SAFETY: as above + if !unsafe { state.input.appendSampleBuffer(©) } { + self.shared.stats.set_error( + format!( + "could not append keep-alive frame: {}", + writer_error(&state) + ), + now_ns, + ); + state.finished = true; + return; + } + + state.last_append_ns = pts_ns; + // not a new frame, and marked so: it keeps the log's idea of the segment's end honest + let _ = writeln!(state.frames_csv, "-,{pts_ns},{now_ns}"); + } + + fn stop(self: Box, reason: EndReason, end_mach_ns: Option) -> DisplayResult { + let mut error = None; + let already_dead = self.shared.stats.error().is_some(); + + // A stream that ScreenCaptureKit has already killed has nothing left to stop, and asking + // would only make us wait out the timeout. + if !already_dead + && let Err(stop_error) = wait_for_completion( + "stopping the screen capture", + STOP_TIMEOUT, + // SAFETY: the stream was started by `start` + |block| unsafe { + self.stream.stopCaptureWithCompletionHandler(Some(block)); + }, + ) + { + error = Some(stop_error.to_string()); + } + + // let any in-flight frame callback finish before the writer is finalized + self.queue.exec_sync(|| {}); + + let end_ns = { + let mut state = self.shared.state.lock(); + state.finished = true; + let _ = state.frames_csv.flush(); + + // Frames only exist when the screen changes, so without an explicit end the file + // would stop at the last change and late timecodes would fall off the end. A segment + // that ended early (a dead stream) ends when it died, never before its last frame. + let now = clock::mono_now_ns(); + let end_ns = end_mach_ns + .unwrap_or(now) + .min(now) + .max(state.last_append_ns); + + if state.started { + // SAFETY: the writer is currently writing + let finished = unsafe { + let ticks = clock::timebase().ns_to_ticks(end_ns); + state + .writer + .endSessionAtSourceTime(CMClock::make_host_time_from_system_units(ticks)); + state.input.markAsFinished(); + let (tx, rx) = mpsc::channel(); + let block = RcBlock::new(move || { + let _ = tx.send(()); + }); + state.writer.finishWritingWithCompletionHandler(&block); + rx.recv_timeout(CALLBACK_TIMEOUT).is_ok() + }; + + if !finished { + error.get_or_insert_with(|| "timed out finalizing the video".to_string()); + } else if let Some(message) = unsafe { state.writer.error() } { + error.get_or_insert_with(|| message.localizedDescription().to_string()); + } + } else { + // SAFETY: the writer never started, so there is nothing to finalize + unsafe { state.writer.cancelWriting() }; + error.get_or_insert_with(|| "no frames were captured".to_string()); + } + + end_ns + }; + + let error = error.or_else(|| self.shared.stats.error()); + self.result(Some(reason), Some(end_ns), error) + } +} + +fn create_writer( + path: &Path, + codec: Codec, + width: usize, + height: usize, +) -> eyre::Result<(Retained, Retained)> { + let path = path + .to_str() + .ok_or_else(|| eyre::Report::msg("session path is not valid UTF-8"))?; + + // SAFETY: all statics are provided by AVFoundation and all arguments are live objects + unsafe { + let url = NSURL::fileURLWithPath(&NSString::from_str(path)); + let file_type = + AVFileTypeMPEG4.ok_or_else(|| eyre::Report::msg("AVFileTypeMPEG4 is unavailable"))?; + let writer = + AVAssetWriter::assetWriterWithURL_fileType_error(&url, file_type).map_err(|error| { + eyre::Report::msg(format!( + "could not create the video writer: {}", + error.localizedDescription() + )) + })?; + + let codec_name: &NSString = match codec { + Codec::Hevc => AVVideoCodecTypeHEVC, + Codec::H264 => AVVideoCodecTypeH264, + } + .ok_or_else(|| eyre::Report::msg("the requested video codec is unavailable"))?; + + let width = NSNumber::new_usize(width); + let height = NSNumber::new_usize(height); + let keys = [ + AVVideoCodecKey.ok_or_else(|| eyre::Report::msg("AVVideoCodecKey is unavailable"))?, + AVVideoWidthKey.ok_or_else(|| eyre::Report::msg("AVVideoWidthKey is unavailable"))?, + AVVideoHeightKey.ok_or_else(|| eyre::Report::msg("AVVideoHeightKey is unavailable"))?, + ]; + let values: [&AnyObject; 3] = [codec_name.as_ref(), width.as_ref(), height.as_ref()]; + let settings: Retained> = + NSDictionary::from_slices(&keys, &values); + + let media_type = + AVMediaTypeVideo.ok_or_else(|| eyre::Report::msg("AVMediaTypeVideo is unavailable"))?; + let input = AVAssetWriterInput::assetWriterInputWithMediaType_outputSettings( + media_type, + Some(&settings), + ); + input.setExpectsMediaDataInRealTime(true); + + // Write the file as movie fragments so that if the recorder is killed or crashes, a + // display is yanked mid-write, or the machine loses power, everything up to the last + // fragment is still a playable video instead of an unfinalized file with no index. + writer.setMovieFragmentInterval(CMTime::new(FRAGMENT_SECONDS, 1)); + + if !writer.canAddInput(&input) { + eyre::bail!("the video writer rejected the input configuration ({codec})"); + } + + writer.addInput(&input); + Ok((writer, input)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn every_recorder_process_is_left_out_of_the_capture_and_nothing_else() { + // this process, by pid + assert!(is_recorder_application(42, "whatever", 42)); + // another recorder (a viewer opened beside a recording), by name + assert!(is_recorder_application(7, "komorebi-recorder", 42)); + assert!(is_recorder_application(7, "Komorebi-Recorder", 42)); + // other applications, including komorebi itself and similarly named ones + assert!(!is_recorder_application(7, "komorebi", 42)); + assert!(!is_recorder_application(7, "komorebi-recorder-helper", 42)); + assert!(!is_recorder_application(7, "Ghostty", 42)); + } +} diff --git a/komorebi-recorder/src/capture/segment.rs b/komorebi-recorder/src/capture/segment.rs new file mode 100644 index 00000000..d8c2ae9a --- /dev/null +++ b/komorebi-recorder/src/capture/segment.rs @@ -0,0 +1,571 @@ +//! The parts of recording one segment that do not depend on which OS captured it. +//! +//! They are where each frame is placed in the file's time, the log of when each arrived, when an +//! idle display needs a frame repeated to keep its file moving, and how to size the encoding. +//! +//! A backend feeds frames in with their timestamps on the master clock and gets back where to put +//! them, so the rules that make a segment's file and its frame log agree are in one place and are +//! tested without a screen. +//! +//! | item | what it decides | +//! |---|---| +//! | [`Placement`] | where each frame, and each repeat of the last frame, goes in the file's time, and that the encoder never sees a time that is not later than the last | +//! | [`FrameLog`] | the `frames-[-].csv` file: when each frame was shown and received, and a row for each repeat | +//! | [`keepalive_due`], [`KEEPALIVE_AFTER`], [`KEEPALIVE_SOURCE_TIMEOUT_NS`] | when a display that has shown nothing new needs its last frame repeated | +//! | [`Latency`] | the clock-unification check, as [`LatencyStats`] | +//! | [`even`], [`h264_bitrate`], [`rebased_100ns`] | the encoding's size and bitrate, and the file's unit of time | +//! +//! # Who uses it +//! +//! The Windows backend (`wgc`) uses all of it. The macOS backend (`sck`) uses only [`even`] and +//! [`Latency`]: it applies the same placement, frame-log and keep-alive rules inline, with the same +//! numbers and private copies of the constants. The unit tests here therefore cover what +//! the Windows backend relies on, and the macOS equivalents are not covered by them. +//! +//! # Time +//! +//! Every `*_ns` argument and result is a time on the master clock (see [`crate::clock`]): +//! nanoseconds on the host's monotonic clock. A time in the file itself, as in +//! [`Placed::file_time`], is in the 100 ns units Media Foundation counts in, measured from the +//! segment's first frame. + +use crate::capture::LatencyStats; +use std::fs::File; +use std::io::BufWriter; +use std::io::Write; +use std::path::Path; +use std::time::Duration; +use std::time::Instant; + +/// How long a display must have shown nothing new before its last frame is repeated: 500 ms. +/// +/// The repeat means a crash would lose only what is in the encoder's buffers: a screen that never +/// changed after the first second would otherwise leave a file that never got past its first +/// fragment. Repeating a frame costs a few bytes (it is identical). See [`keepalive_due`]. +pub const KEEPALIVE_AFTER: Duration = Duration::from_millis(500); + +/// How long a source may be silent and still be kept alive, in nanoseconds: 2 seconds. +/// +/// A source silent for longer than this is not kept alive: repeating the last frame of a capture +/// that has silently died would make a frozen video look healthy while the screen moves on. See +/// [`keepalive_due`]. +pub const KEEPALIVE_SOURCE_TIMEOUT_NS: u64 = 2_000_000_000; + +/// How often the frame log is flushed while frames arrive: one second. +/// +/// This bounds what a crash can lose from the log. See [`FrameLog`] for what is not covered. +pub const LOG_FLUSH_INTERVAL: Duration = Duration::from_secs(1); + +/// How much later than the last thing written the next must be, at the least. +const MIN_STEP_NS: u64 = 1_000_000; + +/// Returns `value` rounded to a whole number and then down to an even number, and never less than +/// two: video codecs want even sizes. +/// +/// # Example +/// +/// ```rust +/// use komorebi_recorder::capture::segment::even; +/// +/// assert_eq!(even(1920.0), 1920); +/// assert_eq!(even(1919.0), 1918); +/// assert_eq!(even(1920.6), 1920); +/// assert_eq!(even(1921.6), 1922); +/// // Never below two, whatever it is given. +/// assert_eq!(even(1.0), 2); +/// assert_eq!(even(0.0), 2); +/// assert_eq!(even(-640.0), 2); +/// ``` +#[must_use] +pub fn even(value: f64) -> usize { + let value = value.round().max(2.0) as usize; + value & !1 +} + +/// Returns the bitrate to encode a screen of this size at, in bits per second. +/// +/// Screen content is mostly flat and changes in bursts, so this is about a tenth of a bit per pixel +/// per frame at the frame rate cap, which is generous for text and keeps a 4K display under a +/// hardware encoder's usual limit. The result is clamped to between 1.5 and 40 megabits per +/// second, and an `fps` of `0` counts as `1`. +/// +/// Only the Windows backend calls it: the macOS backend does not set a bitrate. +/// +/// # Example +/// +/// ```rust +/// use komorebi_recorder::capture::segment::h264_bitrate; +/// +/// // A tenth of a bit per pixel per frame: 1920 * 1080 * 30 * 0.1. +/// assert_eq!(h264_bitrate(1920, 1080, 30), 6_220_800); +/// assert!(h264_bitrate(3840, 2160, 30) > h264_bitrate(1920, 1080, 30)); +/// +/// // A floor for a tiny screen and a ceiling for a huge one. +/// assert_eq!(h264_bitrate(100, 100, 1), 1_500_000); +/// assert_eq!(h264_bitrate(7680, 4320, 60), 40_000_000); +/// ``` +#[must_use] +pub fn h264_bitrate(width: usize, height: usize, fps: u32) -> u32 { + let wanted = width as f64 * height as f64 * f64::from(fps.max(1)) * 0.1; + wanted.clamp(1_500_000.0, 40_000_000.0) as u32 +} + +/// Returns how long after `first_pts_ns` the time `pts_ns` is, in the 100 ns units Media Foundation +/// counts in. +/// +/// This is the time a frame has in the segment's file, whose time zero is its first frame. A +/// `pts_ns` that is not later than `first_pts_ns` gives `0`; a fraction of a unit is dropped. +/// +/// # Example +/// +/// ```rust +/// use komorebi_recorder::capture::segment::rebased_100ns; +/// +/// // 33 ms after the first frame is 330 000 units of 100 ns. +/// assert_eq!(rebased_100ns(5_033_000_000, 5_000_000_000), 330_000); +/// assert_eq!(rebased_100ns(5_000_000_000, 5_000_000_000), 0); +/// assert_eq!(rebased_100ns(4_000_000_000, 5_000_000_000), 0); +/// ``` +#[must_use] +pub fn rebased_100ns(pts_ns: u64, first_pts_ns: u64) -> i64 { + i64::try_from(pts_ns.saturating_sub(first_pts_ns) / 100).unwrap_or(i64::MAX) +} + +/// Where each thing appended to a segment goes in time. +/// +/// The encoder rejects a frame that is not later than the one before it. Frames carry the time the +/// display produced them, which runs slightly ahead of when they are received, so a keep-alive +/// stamped "now" can land a hair after the next real frame; the real frame is moved just past it +/// rather than losing it or the segment. +/// +/// It is guaranteed that every [`Placed::pts_ns`] it returns is later than the one before. A +/// frame that is not later than the last thing placed is moved to 1 ms after it, and a repeat is +/// never less than 1 ms after it. The first frame defines the start of the segment and is never +/// moved. A `Placement` is per segment: start a new one for the next. +/// +/// # Example +/// +/// ```rust +/// use komorebi_recorder::capture::segment::Placement; +/// +/// let mut placement = Placement::default(); +/// assert_eq!(placement.first_ns(), None); +/// +/// // The first frame is time zero of the file; 33 ms later is 330 000 units of 100 ns. +/// let first = placement.frame(5_000_000_000); +/// assert_eq!(first.file_time, 0); +/// let next = placement.frame(5_033_000_000); +/// assert_eq!(next.file_time, 330_000); +/// assert!(!next.moved); +/// +/// // A repeat stamped "now" ran a hair ahead of the next real frame, which is moved just past it +/// // instead of being rejected. +/// placement.repeat(5_600_000_000); +/// let late = placement.frame(5_599_000_000); +/// assert!(late.moved); +/// assert_eq!(late.pts_ns, 5_601_000_000); +/// assert_eq!(placement.last_ns(), 5_601_000_000); +/// ``` +#[derive(Debug, Default)] +pub struct Placement { + first_ns: Option, + last_ns: u64, +} + +/// Where something was placed: what [`Placement::frame`] and [`Placement::repeat`] return. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Placed { + /// On the master clock, after any moving to keep it in order. This is the time to log. + pub pts_ns: u64, + /// Within the segment's file, in 100 ns units: zero for the first. This is the time to hand to + /// the encoder. + pub file_time: i64, + /// Whether it was moved from the time it was given. + pub moved: bool, +} + +impl Placement { + /// The master-clock time of the segment's first frame, once it has one. + #[must_use] + pub fn first_ns(&self) -> Option { + self.first_ns + } + + /// The master-clock time of the last thing appended (`0` before any). + #[must_use] + pub fn last_ns(&self) -> u64 { + self.last_ns + } + + /// Places a real frame produced at `pts_ns`. The first one defines the start of the segment. + /// + /// A frame that is not later than the last thing placed is moved to 1 ms after it + /// ([`Placed::moved`] is then `true`). + pub fn frame(&mut self, pts_ns: u64) -> Placed { + let first = *self.first_ns.get_or_insert(pts_ns); + let adjusted = if self.last_ns > 0 && pts_ns <= self.last_ns { + self.last_ns + MIN_STEP_NS + } else { + pts_ns + }; + + self.last_ns = adjusted; + Placed { + pts_ns: adjusted, + file_time: rebased_100ns(adjusted, first), + moved: adjusted != pts_ns, + } + } + + /// Places a repeat of the last frame at `now_ns`. `None` before there is a first frame to + /// repeat. + /// + /// A repeat is never earlier than 1 ms after the last thing placed, so one asked for "now" + /// right after a frame that ran ahead of the clock is moved later ([`Placed::moved`]). + pub fn repeat(&mut self, now_ns: u64) -> Option { + let first = self.first_ns?; + let adjusted = now_ns.max(self.last_ns + MIN_STEP_NS); + self.last_ns = adjusted; + Some(Placed { + pts_ns: adjusted, + file_time: rebased_100ns(adjusted, first), + moved: adjusted != now_ns, + }) + } +} + +/// Returns `true` if and only if an idle segment is due a repeated frame. +/// +/// Only while the source is demonstrably alive (`source_signal_ns` is when it last did anything, +/// `None` if the backend cannot tell, which counts as alive) and the segment has been still for +/// [`KEEPALIVE_AFTER`]. A source silent for [`KEEPALIVE_SOURCE_TIMEOUT_NS`] or longer is not alive. +/// `last_append_ns` is the master-clock time of the last thing written to the segment, as +/// [`Placement::last_ns`] reports it. +/// +/// # Example +/// +/// ```rust +/// use komorebi_recorder::capture::segment::keepalive_due; +/// +/// let half_second = 500_000_000; +/// // Still for half a second, and the source is alive or cannot say. +/// assert!(keepalive_due(half_second, 0, None)); +/// assert!(keepalive_due(half_second, 0, Some(half_second - 1))); +/// // Not still for long enough. +/// assert!(!keepalive_due(half_second - 1, 0, None)); +/// // The source has been silent for 5 seconds: a dead capture is not made to look alive. +/// assert!(!keepalive_due(5_000_000_000, 0, Some(0))); +/// ``` +#[must_use] +pub fn keepalive_due(now_ns: u64, last_append_ns: u64, source_signal_ns: Option) -> bool { + let alive = source_signal_ns + .is_none_or(|last| now_ns.saturating_sub(last) < KEEPALIVE_SOURCE_TIMEOUT_NS); + let still_for = now_ns.saturating_sub(last_append_ns); + alive && still_for >= u64::try_from(KEEPALIVE_AFTER.as_nanos()).unwrap_or(u64::MAX) +} + +/// Accumulates how far the frames' times lag behind the moment they were received. +/// +/// If the frames were not on the host clock this would be wildly off, which makes it the +/// clock-unification check. [`Latency::finish`] reads out a [`LatencyStats`] without ending the +/// accumulation, so a segment can be summarized while it is still recording. +/// +/// # Example +/// +/// ```rust +/// use komorebi_recorder::capture::segment::Latency; +/// +/// let mut latency = Latency::default(); +/// // No frames, nothing to doubt. +/// assert!(latency.finish().looks_like_host_clock()); +/// +/// // Received minus stamped, in nanoseconds: the first frame was stamped 5 ms ahead. +/// for ns in [-5_000_000, 10_000_000, 30_000_000] { +/// latency.record(ns); +/// } +/// let stats = latency.finish(); +/// assert_eq!((stats.samples, stats.min_ns, stats.max_ns), (3, -5_000_000, 30_000_000)); +/// assert_eq!(stats.mean_ns, 11_666_666); +/// assert!(stats.looks_like_host_clock()); +/// ``` +#[derive(Default)] +pub struct Latency { + samples: u64, + min_ns: i64, + max_ns: i64, + sum_ns: i128, +} + +impl Latency { + /// Adds one frame's latency: the time it was received minus the time it is stamped with, in + /// nanoseconds. Negative when the stamp is ahead of the arrival. + pub fn record(&mut self, latency_ns: i64) { + if self.samples == 0 { + self.min_ns = latency_ns; + self.max_ns = latency_ns; + } else { + self.min_ns = self.min_ns.min(latency_ns); + self.max_ns = self.max_ns.max(latency_ns); + } + + self.samples += 1; + self.sum_ns += i128::from(latency_ns); + } + + /// Returns the statistics of the latencies recorded so far. With none, all zeros. + #[must_use] + pub fn finish(&self) -> LatencyStats { + LatencyStats { + samples: self.samples, + min_ns: self.min_ns, + max_ns: self.max_ns, + mean_ns: if self.samples == 0 { + 0 + } else { + (self.sum_ns / i128::from(self.samples)) as i64 + }, + } + } +} + +/// `frames-[-].csv`: every frame's time on the master clock and when it was received, +/// and a `-` row for each repeat. +/// +/// It is authoritative for when each frame was shown, whatever the file's own times say. +/// +/// There is one row per line, `,,`, with both times in nanoseconds on +/// the master clock. A repeat of the last frame, which keeps the file moving while the display is +/// still, has `-` in place of an index and is not counted as a frame: +/// +/// ```text +/// 0,5000000000,5000900000 +/// 1,5033000000,5034000000 +/// -,5600000000,5600000000 +/// ``` +/// +/// [`FrameLog::create`] writes no header row. The reader of the file (`recovery`) skips the first +/// line of the file as a header; the macOS backend writes `index,pts_mach_ns,received_mach_ns` +/// there itself, and the Windows backend does not. +/// +/// # Durability +/// +/// Rows are buffered. [`FrameLog::frame`] flushes when [`LOG_FLUSH_INTERVAL`] has passed since the +/// last flush, and [`FrameLog::flush`] flushes at once. [`FrameLog::repeat`] does not flush, so +/// while a display is still the rows for its repeats stay in memory until the next real frame or +/// [`FrameLog::flush`]. A crash can lose what is still buffered, and can leave the last row of the +/// file cut short if the buffer filled in the middle of it, so a reader has to ignore a row that +/// does not parse (`recovery` does). A write that fails is ignored: the video is the source of +/// truth. +/// +/// # Example +/// +/// ```rust +/// use komorebi_recorder::capture::segment::FrameLog; +/// +/// let path = std::env::temp_dir() +/// .join(format!("komorebi-recorder-doc-frame-log-{}.csv", std::process::id())); +/// let mut log = FrameLog::create(&path)?; +/// log.frame(100, 105); // stamped 100, received at 105 +/// log.repeat(700, 710); // the display was still: the last frame repeated +/// log.frame(900, 903); +/// log.flush(); +/// +/// assert_eq!(log.frames(), 2, "a repeat is not a frame"); +/// assert_eq!(std::fs::read_to_string(&path)?, "0,100,105\n-,700,710\n1,900,903\n"); +/// # std::fs::remove_file(&path)?; +/// # Ok::<(), Box>(()) +/// ``` +pub struct FrameLog { + out: BufWriter, + last_flush: Instant, + index: u64, +} + +impl FrameLog { + /// Creates the log at `path`, truncating a file that is already there. Nothing is written + /// until the first row. + /// + /// # Errors + /// + /// Returns the I/O error if the file cannot be created: the directory does not exist, or the + /// process may not write there. + pub fn create(path: &Path) -> std::io::Result { + Ok(Self { + out: BufWriter::new(File::create(path)?), + last_flush: Instant::now(), + index: 0, + }) + } + + /// Returns how many real frames have been logged. Repeats are not counted. + #[must_use] + pub fn frames(&self) -> u64 { + self.index + } + + /// Logs a real frame stamped `pts_ns` and received at `received_ns`, both on the master clock. + /// + /// A failed write must not take down the capture: the video is the source of truth and this is + /// how it is read. So a failed write is not reported and the row is lost, and the frame is + /// still counted by [`FrameLog::frames`]. + pub fn frame(&mut self, pts_ns: u64, received_ns: u64) { + let _ = writeln!(self.out, "{},{pts_ns},{received_ns}", self.index); + self.index += 1; + self.flush_if_due(); + } + + /// Logs a repeat of the last frame at `pts_ns`, made at `now_ns`: not a new frame, and marked + /// so, but it keeps the log's idea of where the segment ends honest. + pub fn repeat(&mut self, pts_ns: u64, now_ns: u64) { + let _ = writeln!(self.out, "-,{pts_ns},{now_ns}"); + } + + fn flush_if_due(&mut self) { + if self.last_flush.elapsed() >= LOG_FLUSH_INTERVAL { + // on whole lines, so a crash never leaves a torn row behind + let _ = self.out.flush(); + self.last_flush = Instant::now(); + } + } + + /// Writes what is buffered to the file. A failed write is ignored, as in [`FrameLog::frame`]. + pub fn flush(&mut self) { + let _ = self.out.flush(); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn sizes_are_even_and_never_below_two() { + assert_eq!(even(1920.0), 1920); + assert_eq!(even(1919.0), 1918); + assert_eq!(even(1920.6), 1920); + assert_eq!(even(0.0), 2); + assert_eq!(even(1.0), 2); + } + + #[test] + fn the_bitrate_follows_the_pixels_within_what_an_encoder_takes() { + let hd = h264_bitrate(1920, 1080, 30); + let uhd = h264_bitrate(3840, 2160, 30); + assert!((5_000_000..10_000_000).contains(&hd), "{hd}"); + assert!(uhd > hd); + assert_eq!( + h264_bitrate(100, 100, 1), + 1_500_000, + "a floor for a tiny one" + ); + assert_eq!( + h264_bitrate(7680, 4320, 60), + 40_000_000, + "a ceiling for a huge one" + ); + } + + #[test] + fn the_first_frame_is_time_zero_and_the_rest_follow_it() { + let mut placement = Placement::default(); + let first = placement.frame(5_000_000_000); + assert_eq!(first.file_time, 0); + assert_eq!(placement.first_ns(), Some(5_000_000_000)); + + let next = placement.frame(5_033_000_000); + assert_eq!( + next.file_time, 330_000, + "33 ms is 330000 hundred-nanosecond units" + ); + assert!(!next.moved); + } + + #[test] + fn a_frame_not_later_than_the_last_is_moved_just_past_it() { + let mut placement = Placement::default(); + placement.frame(1_000_000_000); + // a repeat stamped "now" ran a hair ahead of the next real frame + placement.repeat(1_600_000_000); + let late = placement.frame(1_599_000_000); + + assert!(late.moved); + assert_eq!(late.pts_ns, 1_601_000_000); + assert!(late.file_time > 0); + } + + #[test] + fn every_placement_is_strictly_later_than_the_one_before() { + let mut placement = Placement::default(); + let mut last = 0; + for pts in [10, 20, 20, 15, 30, 30, 31] { + let placed = placement.frame(pts * 1_000_000); + assert!( + placed.pts_ns > last || last == 0, + "{pts}: {placed:?} after {last}" + ); + last = placed.pts_ns; + } + } + + #[test] + fn nothing_can_be_repeated_before_the_first_frame() { + assert_eq!(Placement::default().repeat(1_000_000_000), None); + } + + #[test] + fn a_repeat_is_never_earlier_than_what_it_repeats() { + let mut placement = Placement::default(); + placement.frame(2_000_000_000); + let repeat = placement.repeat(1_000_000_000).expect("there is a frame"); + assert!(repeat.pts_ns > 2_000_000_000); + } + + #[test] + fn a_still_display_is_kept_alive_only_while_its_source_is() { + let half = 500_000_000; + // still for long enough, source alive or unable to say + assert!(keepalive_due(2 * half, 0, None)); + assert!(keepalive_due(2 * half, 0, Some(2 * half - 1))); + // not still for long enough + assert!(!keepalive_due(half - 1, 0, None)); + // source silent for over two seconds: a dead capture is not made to look alive + assert!(!keepalive_due(10 * half, 0, Some(0))); + } + + #[test] + fn latency_tracks_the_range_and_mean_and_says_what_clock_it_is() { + let mut latency = Latency::default(); + assert!( + latency.finish().looks_like_host_clock(), + "no frames, nothing to doubt" + ); + for ns in [-5_000_000, 10_000_000, 30_000_000] { + latency.record(ns); + } + let stats = latency.finish(); + assert_eq!( + (stats.samples, stats.min_ns, stats.max_ns), + (3, -5_000_000, 30_000_000) + ); + assert_eq!(stats.mean_ns, 11_666_666); + assert!(stats.looks_like_host_clock()); + } + + #[test] + fn the_frame_log_writes_frames_and_marks_repeats() { + let path = + std::env::temp_dir().join(format!("komorebi-framelog-{}.csv", std::process::id())); + let mut log = FrameLog::create(&path).expect("create"); + log.frame(100, 105); + log.repeat(700, 710); + log.frame(900, 903); + log.flush(); + + assert_eq!(log.frames(), 2, "a repeat is not a frame"); + assert_eq!( + std::fs::read_to_string(&path).expect("read"), + "0,100,105\n-,700,710\n1,900,903\n" + ); + let _ = std::fs::remove_file(&path); + } +} diff --git a/komorebi-recorder/src/capture/wgc.rs b/komorebi-recorder/src/capture/wgc.rs new file mode 100644 index 00000000..639c4f73 --- /dev/null +++ b/komorebi-recorder/src/capture/wgc.rs @@ -0,0 +1,860 @@ +//! The Windows capture backend: Windows.Graphics.Capture into a Media Foundation fragmented MP4. +//! +//! This module exists on Windows only. [`WgcBackend`] is the [`CaptureBackend`] and +//! [`DisplayCapture`] the [`RunningCapture`]; [`crate::platform::capture`] re-exports `WgcBackend` +//! as the host's `Backend`. +//! +//! One [`DisplayCapture`] per monitor. Windows.Graphics.Capture hands frames over on a thread pool +//! thread of its own whenever the picture changes (and not otherwise: a still screen is silent), +//! each stamped with the time it was produced on the performance counter, which is the master +//! clock. Each frame is copied to a texture of ours (the capture's own is recycled as soon as the +//! frame is closed, and the encoder reads asynchronously) and written to the sink writer, which +//! encodes on the GPU where it can. The timestamps in the file are rebased to start at zero with +//! the segment; the exact master-clock time of every frame is in `frames-.csv`, as on +//! macOS, and that is what is trusted for when a frame was shown. +//! +//! # How a segment is recorded +//! +//! 1. [`DisplayCapture::start`] makes a capture item for the monitor, creates one Direct3D 11 +//! device that the capture and the encoder share, and creates the sink writer: H.264 in a +//! fragmented MP4 with hardware transforms enabled, throttling off, fragments about a second +//! long and a key frame about every second, at the bitrate [`h264_bitrate`] gives for the +//! size. It creates the frame log, a pool of three frames and the capture session, and starts +//! it. +//! 2. Each frame is copied to a texture the capture does not recycle, placed in time by a +//! [`Placement`], written to the +//! sink writer with the nominal duration of one frame, and logged by a [`FrameLog`]. A frame +//! whose content size differs from the display's size at the start ends the capture with an +//! error: the display changed size. +//! 3. [`RunningCapture::keepalive`] first checks whether the GPU was reset or removed, which ends +//! the capture with an error. Otherwise it repeats the last frame when the display has shown +//! nothing new for [`KEEPALIVE_AFTER`](crate::capture::segment::KEEPALIVE_AFTER) +//! ([`keepalive_due`], with a source that is always taken to be alive). +//! 4. [`RunningCapture::stop`] closes the session and the pool, then takes the lock that frame +//! callbacks hold, so one in flight finishes first. A still display has produced no frames, so +//! the segment ends with a repeat of the last frame at the segment's end time, and the sink +//! writer is finalized. +//! +//! # What this backend does not do +//! +//! - It records H.264 only: [`Codec::Hevc`] is refused when a capture starts, because Media +//! Foundation's MP4 writer does not write it. +//! - It reports no sign of life ([`RunningCapture::last_signal_mach_ns`] is `None`), because +//! Windows.Graphics.Capture sends no callbacks while the picture does not change. A stalled +//! capture is not told by silence. A display that went away is told by its capture item closing, +//! a GPU that went away by its device, and either by the display poll. +//! - It does not count idle callbacks or dropped frames: [`DisplayResult::idle_callbacks`] and +//! [`DisplayResult::dropped`] are always `0`. +//! +//! # Status +//! +//! Written from the documentation of the APIs. At the time of writing (2026-10-27) it type-checks +//! against the real bindings and passes clippy for the Windows target, but has not been run on a +//! Windows machine. What is tested is the placement and logging rules it relies on +//! ([`crate::capture::segment`]). `doctor --probe capture` is how it is checked on a real machine +//! (checkpoint W1 in `docs/dev/windows-checkpoints.md`), including a recording that is killed and +//! a static screen. + +use super::CaptureBackend; +use super::CaptureConfig; +use super::CaptureStats; +use super::Codec; +use super::DisplayDescriptor; +use super::DisplayResult; +use super::EndReason; +use super::RunningCapture; +use super::frames_file_name; +use super::segment::FrameLog; +use super::segment::Latency; +use super::segment::Placement; +use super::segment::even; +use super::segment::h264_bitrate; +use super::segment::keepalive_due; +use super::video_file_name; +use crate::clock; +use crate::platform::media::pack; +use color_eyre::eyre; +use color_eyre::eyre::WrapErr; +use parking_lot::Mutex; +use std::path::Path; +use std::sync::Arc; +use std::sync::atomic::Ordering; +use windows::Foundation::TypedEventHandler; +use windows::Graphics::Capture::Direct3D11CaptureFramePool; +use windows::Graphics::Capture::GraphicsCaptureItem; +use windows::Graphics::Capture::GraphicsCaptureSession; +use windows::Graphics::DirectX::Direct3D11::IDirect3DDevice; +use windows::Graphics::DirectX::DirectXPixelFormat; +use windows::Graphics::SizeInt32; +use windows::Win32::Foundation::HMODULE; +use windows::Win32::Graphics::Direct3D::D3D_DRIVER_TYPE_HARDWARE; +use windows::Win32::Graphics::Direct3D::D3D_FEATURE_LEVEL_11_0; +use windows::Win32::Graphics::Direct3D11::D3D11_BIND_RENDER_TARGET; +use windows::Win32::Graphics::Direct3D11::D3D11_BIND_SHADER_RESOURCE; +use windows::Win32::Graphics::Direct3D11::D3D11_BOX; +use windows::Win32::Graphics::Direct3D11::D3D11_CREATE_DEVICE_BGRA_SUPPORT; +use windows::Win32::Graphics::Direct3D11::D3D11_CREATE_DEVICE_VIDEO_SUPPORT; +use windows::Win32::Graphics::Direct3D11::D3D11_SDK_VERSION; +use windows::Win32::Graphics::Direct3D11::D3D11_TEXTURE2D_DESC; +use windows::Win32::Graphics::Direct3D11::D3D11_USAGE_DEFAULT; +use windows::Win32::Graphics::Direct3D11::D3D11CreateDevice; +use windows::Win32::Graphics::Direct3D11::ID3D11Device; +use windows::Win32::Graphics::Direct3D11::ID3D11DeviceContext; +use windows::Win32::Graphics::Direct3D11::ID3D11Multithread; +use windows::Win32::Graphics::Direct3D11::ID3D11Texture2D; +use windows::Win32::Graphics::Dxgi::Common::DXGI_FORMAT_B8G8R8A8_UNORM; +use windows::Win32::Graphics::Dxgi::Common::DXGI_SAMPLE_DESC; +use windows::Win32::Graphics::Dxgi::IDXGIDevice; +use windows::Win32::Media::MediaFoundation::IMF2DBuffer; +use windows::Win32::Media::MediaFoundation::IMFAttributes; +use windows::Win32::Media::MediaFoundation::IMFDXGIDeviceManager; +use windows::Win32::Media::MediaFoundation::IMFMediaBuffer; +use windows::Win32::Media::MediaFoundation::IMFMediaType; +use windows::Win32::Media::MediaFoundation::IMFSample; +use windows::Win32::Media::MediaFoundation::IMFSinkWriter; +use windows::Win32::Media::MediaFoundation::MF_MPEG4SINK_MIN_FRAGMENT_DURATION; +use windows::Win32::Media::MediaFoundation::MF_MT_AVG_BITRATE; +use windows::Win32::Media::MediaFoundation::MF_MT_FRAME_RATE; +use windows::Win32::Media::MediaFoundation::MF_MT_FRAME_SIZE; +use windows::Win32::Media::MediaFoundation::MF_MT_INTERLACE_MODE; +use windows::Win32::Media::MediaFoundation::MF_MT_MAJOR_TYPE; +use windows::Win32::Media::MediaFoundation::MF_MT_MAX_KEYFRAME_SPACING; +use windows::Win32::Media::MediaFoundation::MF_MT_MPEG2_PROFILE; +use windows::Win32::Media::MediaFoundation::MF_MT_PIXEL_ASPECT_RATIO; +use windows::Win32::Media::MediaFoundation::MF_MT_SUBTYPE; +use windows::Win32::Media::MediaFoundation::MF_READWRITE_ENABLE_HARDWARE_TRANSFORMS; +use windows::Win32::Media::MediaFoundation::MF_SINK_WRITER_D3D_MANAGER; +use windows::Win32::Media::MediaFoundation::MF_SINK_WRITER_DISABLE_THROTTLING; +use windows::Win32::Media::MediaFoundation::MF_TRANSCODE_CONTAINERTYPE; +use windows::Win32::Media::MediaFoundation::MFCreateAttributes; +use windows::Win32::Media::MediaFoundation::MFCreateDXGIDeviceManager; +use windows::Win32::Media::MediaFoundation::MFCreateDXGISurfaceBuffer; +use windows::Win32::Media::MediaFoundation::MFCreateMediaType; +use windows::Win32::Media::MediaFoundation::MFCreateSample; +use windows::Win32::Media::MediaFoundation::MFCreateSinkWriterFromURL; +use windows::Win32::Media::MediaFoundation::MFMediaType_Video; +use windows::Win32::Media::MediaFoundation::MFTranscodeContainerType_FMPEG4; +use windows::Win32::Media::MediaFoundation::MFVideoFormat_ARGB32; +use windows::Win32::Media::MediaFoundation::MFVideoFormat_H264; +use windows::Win32::Media::MediaFoundation::MFVideoInterlace_Progressive; +use windows::Win32::Media::MediaFoundation::eAVEncH264VProfile_High; +use windows::Win32::System::WinRT::Direct3D11::CreateDirect3D11DeviceFromDXGIDevice; +use windows::Win32::System::WinRT::Direct3D11::IDirect3DDxgiInterfaceAccess; +use windows::Win32::System::WinRT::Graphics::Capture::IGraphicsCaptureItemInterop; +use windows::core::Interface; +use windows::core::PCWSTR; + +/// How many frames the capture's pool holds: enough that a frame being copied does not make the +/// next one wait. +const POOL_FRAMES: i32 = 3; + +/// A fragment of the file is closed about this often (in 100 ns units): what a crash can lose. +const FRAGMENT_100NS: u64 = 10_000_000; + +/// The GPU a capture works on: one device for the capture and the encoder to share. +struct Gpu { + device: ID3D11Device, + context: ID3D11DeviceContext, + /// The same device as Windows Runtime sees it, which the frame pool wants. + winrt: IDirect3DDevice, + /// Lets the encoder use the same device, so frames stay on the GPU. + manager: IMFDXGIDeviceManager, +} + +impl Gpu { + fn new() -> eyre::Result { + // SAFETY: plain creation calls; every out pointer is valid for the call + unsafe { + let (mut device, mut context) = (None, None); + D3D11CreateDevice( + None, + D3D_DRIVER_TYPE_HARDWARE, + HMODULE::default(), + D3D11_CREATE_DEVICE_BGRA_SUPPORT | D3D11_CREATE_DEVICE_VIDEO_SUPPORT, + Some(&[D3D_FEATURE_LEVEL_11_0]), + D3D11_SDK_VERSION, + Some(&raw mut device), + None, + Some(&raw mut context), + ) + .wrap_err("could not create a Direct3D 11 device")?; + let device = device.ok_or_else(|| eyre::Report::msg("no Direct3D 11 device"))?; + let context = context.ok_or_else(|| eyre::Report::msg("no Direct3D 11 context"))?; + + // the capture, the copy and the encoder all use this device, from several threads + let multithread: ID3D11Multithread = device.cast()?; + let _ = multithread.SetMultithreadProtected(true); + + let dxgi: IDXGIDevice = device.cast()?; + let winrt: IDirect3DDevice = CreateDirect3D11DeviceFromDXGIDevice(&dxgi) + .wrap_err("could not wrap the device for Windows.Graphics.Capture")? + .cast()?; + + let (mut token, mut manager) = (0_u32, None); + MFCreateDXGIDeviceManager(&raw mut token, &raw mut manager) + .wrap_err("could not create the encoder's device manager")?; + let manager = + manager.ok_or_else(|| eyre::Report::msg("no Media Foundation device manager"))?; + manager + .ResetDevice(&device, token) + .wrap_err("could not give the device to the encoder")?; + + Ok(Self { + device, + context, + winrt, + manager, + }) + } + } + + /// Whether the GPU has been reset or removed (a driver update, an undock that took the + /// discrete GPU): every frame after that is garbage. + fn removed(&self) -> Option { + // SAFETY: no preconditions + let reason = unsafe { self.device.GetDeviceRemovedReason() }; + reason + .is_err() + .then(|| format!("the GPU was reset or removed ({reason:?})")) + } + + /// A texture of our own holding the top left `width` x `height` pixels of `source`, which is + /// the capture's and goes back to its pool when the frame is closed. + fn copy( + &self, + source: &ID3D11Texture2D, + width: u32, + height: u32, + ) -> eyre::Result { + let description = D3D11_TEXTURE2D_DESC { + Width: width, + Height: height, + MipLevels: 1, + ArraySize: 1, + Format: DXGI_FORMAT_B8G8R8A8_UNORM, + SampleDesc: DXGI_SAMPLE_DESC { + Count: 1, + Quality: 0, + }, + Usage: D3D11_USAGE_DEFAULT, + BindFlags: (D3D11_BIND_RENDER_TARGET.0 | D3D11_BIND_SHADER_RESOURCE.0) as u32, + ..Default::default() + }; + + // SAFETY: the texture is created from a valid description; the copy is between textures + // of the same device, and the immediate context is protected for use from many threads + unsafe { + let mut texture = None; + self.device + .CreateTexture2D(&raw const description, None, Some(&raw mut texture)) + .wrap_err("could not allocate a frame")?; + let texture = texture.ok_or_else(|| eyre::Report::msg("no frame texture"))?; + + let region = D3D11_BOX { + left: 0, + top: 0, + front: 0, + right: width, + bottom: height, + back: 1, + }; + self.context.CopySubresourceRegion( + &texture, + 0, + 0, + 0, + 0, + source, + 0, + Some(&raw const region), + ); + Ok(texture) + } + } +} + +/// An encoded sink for one segment: H.264 in a fragmented MP4. +struct Writer { + sink: IMFSinkWriter, + stream: u32, + /// How long one frame at the nominal rate lasts, in 100 ns units: what the last frame of a + /// still display is given. + frame_100ns: i64, +} + +fn input_type(width: u32, height: u32, fps: u32) -> eyre::Result { + // SAFETY: attribute setters on a media type just created + unsafe { + let media = MFCreateMediaType()?; + media.SetGUID(&MF_MT_MAJOR_TYPE, &MFMediaType_Video)?; + media.SetGUID(&MF_MT_SUBTYPE, &MFVideoFormat_ARGB32)?; + media.SetUINT32(&MF_MT_INTERLACE_MODE, MFVideoInterlace_Progressive.0 as u32)?; + media.SetUINT64(&MF_MT_FRAME_SIZE, pack(width, height))?; + media.SetUINT64(&MF_MT_FRAME_RATE, pack(fps, 1))?; + media.SetUINT64(&MF_MT_PIXEL_ASPECT_RATIO, pack(1, 1))?; + Ok(media) + } +} + +fn output_type(width: u32, height: u32, fps: u32) -> eyre::Result { + let bitrate = h264_bitrate(width as usize, height as usize, fps); + // SAFETY: attribute setters on a media type just created + unsafe { + let media = MFCreateMediaType()?; + media.SetGUID(&MF_MT_MAJOR_TYPE, &MFMediaType_Video)?; + media.SetGUID(&MF_MT_SUBTYPE, &MFVideoFormat_H264)?; + media.SetUINT32(&MF_MT_AVG_BITRATE, bitrate)?; + media.SetUINT32(&MF_MT_INTERLACE_MODE, MFVideoInterlace_Progressive.0 as u32)?; + media.SetUINT64(&MF_MT_FRAME_SIZE, pack(width, height))?; + media.SetUINT64(&MF_MT_FRAME_RATE, pack(fps, 1))?; + media.SetUINT64(&MF_MT_PIXEL_ASPECT_RATIO, pack(1, 1))?; + media.SetUINT32(&MF_MT_MPEG2_PROFILE, eAVEncH264VProfile_High.0 as u32)?; + // a key frame about every second: the fragmented file is cut into fragments at key + // frames, so this is how much of it a crash can take + media.SetUINT32(&MF_MT_MAX_KEYFRAME_SPACING, fps)?; + Ok(media) + } +} + +impl Writer { + fn create(path: &Path, gpu: &Gpu, width: u32, height: u32, fps: u32) -> eyre::Result { + let fps = fps.max(1); + let wide: Vec = path + .to_string_lossy() + .encode_utf16() + .chain(Some(0)) + .collect(); + + // SAFETY: attribute setters and creation calls with the arguments they document; `wide` + // outlives the call that reads it + unsafe { + let mut attributes: Option = None; + MFCreateAttributes(&raw mut attributes, 6)?; + let attributes = + attributes.ok_or_else(|| eyre::Report::msg("no Media Foundation attributes"))?; + + // the encoder works on the GPU's textures, on the GPU where it can + attributes.SetUnknown(&MF_SINK_WRITER_D3D_MANAGER, &gpu.manager)?; + attributes.SetUINT32(&MF_READWRITE_ENABLE_HARDWARE_TRANSFORMS, 1)?; + // a writer that holds frames back to pace the producer would hold back a recording + attributes.SetUINT32(&MF_SINK_WRITER_DISABLE_THROTTLING, 1)?; + // fragmented, so that what was written before a crash can still be played + attributes.SetGUID( + &MF_TRANSCODE_CONTAINERTYPE, + &MFTranscodeContainerType_FMPEG4, + )?; + attributes.SetUINT64(&MF_MPEG4SINK_MIN_FRAGMENT_DURATION, FRAGMENT_100NS)?; + + let sink = MFCreateSinkWriterFromURL(PCWSTR(wide.as_ptr()), None, &attributes) + .wrap_err_with(|| format!("could not create the video file {}", path.display()))?; + + let stream = sink + .AddStream(&output_type(width, height, fps)?) + .wrap_err("the H.264 encoder does not accept this size")?; + sink.SetInputMediaType(stream, &input_type(width, height, fps)?, None) + .wrap_err("the encoder cannot take frames of this format")?; + sink.BeginWriting() + .wrap_err("could not start writing the video")?; + + Ok(Self { + sink, + stream, + frame_100ns: i64::from(10_000_000 / fps), + }) + } + } + + /// A sample of `buffer` shown from `time` for `duration` (both in 100 ns units). + fn sample(buffer: &IMFMediaBuffer, time: i64, duration: i64) -> eyre::Result { + // SAFETY: a sample just created, given a buffer that is live + unsafe { + let sample = MFCreateSample()?; + sample.AddBuffer(buffer)?; + sample.SetSampleTime(time)?; + sample.SetSampleDuration(duration)?; + Ok(sample) + } + } + + fn write(&self, buffer: &IMFMediaBuffer, time: i64) -> eyre::Result<()> { + let sample = Self::sample(buffer, time, self.frame_100ns)?; + // SAFETY: the writer is writing (`create` began it) and the sample is complete + unsafe { self.sink.WriteSample(self.stream, &sample) } + .wrap_err("the encoder rejected a frame") + } + + fn finish(&self) -> eyre::Result<()> { + // SAFETY: the writer is writing; this waits for what is queued to be encoded and the file + // to be closed + unsafe { self.sink.Finalize() }.wrap_err("could not finish the video file") + } +} + +/// A texture as a Media Foundation buffer the encoder can read without it leaving the GPU. +fn buffer_of(texture: &ID3D11Texture2D) -> eyre::Result { + // SAFETY: the texture is live and has one subresource + unsafe { + let buffer = MFCreateDXGISurfaceBuffer(&ID3D11Texture2D::IID, texture, 0, false)?; + // a surface buffer reports no length until it is told, and the converter in front of the + // encoder reads it + let length = buffer.cast::()?.GetContiguousLength()?; + buffer.SetCurrentLength(length)?; + Ok(buffer) + } +} + +/// Everything a frame callback mutates, behind a lock so that `stop` can finish the file safely. +struct State { + writer: Writer, + placement: Placement, + log: FrameLog, + latency: Latency, + /// The most recent frame, kept so that a still display can have it repeated. + last_frame: Option, + started: bool, + finished: bool, +} + +struct Shared { + stats: Arc, + state: Mutex, + gpu: Gpu, + /// The size the segment is recorded at, in pixels (the source's, made even). + width: u32, + height: u32, + /// The size of the display when the capture began: a frame of any other size means it changed. + source: SizeInt32, +} + +// SAFETY: the Direct3D device is created multithread-protected and free-threaded, the capture's +// callbacks and Media Foundation's sink writer are agile (they work from any thread), and every +// access to the writer, the frame log and the placement is under the `state` mutex. +unsafe impl Send for Shared {} +// SAFETY: see above +unsafe impl Sync for Shared {} + +/// What one frame from the capture does: copy it, place it in time, write it, log it. +fn on_frame(shared: &Shared, pool: &Direct3D11CaptureFramePool) { + let received_ns = clock::mono_now_ns(); + shared.stats.callbacks.fetch_add(1, Ordering::Relaxed); + shared + .stats + .last_callback_mach_ns + .store(received_ns, Ordering::Relaxed); + + // closed pools and a capture that is being stopped answer with an error: nothing to do + let Ok(frame) = pool.TryGetNextFrame() else { + return; + }; + let outcome = frame_into_file(shared, &frame, received_ns); + // hands the capture's buffer back to its pool, whatever happened + let _ = frame.Close(); + + if let Err(error) = outcome { + shared.stats.set_error(format!("{error:#}"), received_ns); + shared.state.lock().finished = true; + } +} + +fn frame_into_file( + shared: &Shared, + frame: &windows::Graphics::Capture::Direct3D11CaptureFrame, + received_ns: u64, +) -> eyre::Result<()> { + let size = frame.ContentSize()?; + if size.Width != shared.source.Width || size.Height != shared.source.Height { + // the display changed size under us (smaller would be cropped wrong, larger would be + // cut off): the manager sees that too, and starts a new segment + eyre::bail!( + "the display changed size to {}x{} while recording {}x{}", + size.Width, + size.Height, + shared.source.Width, + shared.source.Height + ); + } + + // the frame's time on the performance counter, in 100 ns units + let pts_ns = u64::try_from(frame.SystemRelativeTime()?.Duration) + .unwrap_or(0) + .saturating_mul(100); + + let surface = frame.Surface()?; + let access: IDirect3DDxgiInterfaceAccess = surface.cast()?; + // SAFETY: a capture surface is a texture of the capture's device + let source: ID3D11Texture2D = unsafe { access.GetInterface()? }; + let texture = shared.gpu.copy(&source, shared.width, shared.height)?; + let buffer = buffer_of(&texture)?; + + let mut state = shared.state.lock(); + if state.finished { + return Ok(()); + } + + if !state.started { + state.started = true; + shared + .stats + .first_pts_mach_ns + .store(pts_ns, Ordering::Relaxed); + } + + let placed = state.placement.frame(pts_ns); + state.writer.write(&buffer, placed.file_time)?; + state.last_frame = Some(buffer); + state + .latency + .record(received_ns as i64 - placed.pts_ns as i64); + state.log.frame(placed.pts_ns, received_ns); + + shared.stats.frames.fetch_add(1, Ordering::Relaxed); + shared + .stats + .last_pts_mach_ns + .store(placed.pts_ns, Ordering::Relaxed); + Ok(()) +} + +/// The monitor as a capture item. +fn item_for(display: &DisplayDescriptor) -> eyre::Result { + let monitor = crate::platform::capture::monitor_handle(display.id).ok_or_else(|| { + eyre::Report::msg(format!( + "display {} is not available for capture (yet)", + display.id + )) + })?; + + let interop = windows::core::factory::() + .wrap_err("Windows.Graphics.Capture is not available (Windows 10 1803 or later)")?; + // SAFETY: a monitor handle that was attached a moment ago + unsafe { interop.CreateForMonitor(monitor) }.wrap_err("Windows would not capture that display") +} + +/// An in-progress capture of one display: one segment, recorded by Windows.Graphics.Capture into a +/// Media Foundation sink writer. +/// +/// Built by [`DisplayCapture::start`] and used through [`RunningCapture`]. The video is H.264 at +/// the display's size in physical pixels, rounded to even numbers, and [`DisplayResult::scale`] is +/// `1.0`. The first error a capture reports (its item closing, the GPU being removed, a frame of +/// another size, the encoder rejecting a frame) ends the segment as +/// [`EndReason::StreamError`] when [`DisplayManager`](crate::capture::manager::DisplayManager) +/// next polls. [`stop`](RunningCapture::stop) always returns a [`DisplayResult`]: a segment that +/// never received a frame has its empty file removed and the error `no frames were captured`. +pub struct DisplayCapture { + descriptor: DisplayDescriptor, + segment: u32, + width_px: usize, + height_px: usize, + codec: Codec, + file: String, + frames_file: String, + path: std::path::PathBuf, + /// Kept alive for as long as the capture runs: its `Closed` event is how a removed display is + /// noticed. + _item: GraphicsCaptureItem, + pool: Direct3D11CaptureFramePool, + session: GraphicsCaptureSession, + shared: Arc, +} + +impl DisplayCapture { + /// Starts capturing `display` into `dir` as the given segment. + /// + /// Returns once the capture session has started, without waiting for a frame: + /// `video_t0_mach_ns` is `None` until one arrives. The video is the display's size in physical + /// pixels rounded to even numbers (a frame is cropped to the top left of the display if the + /// display's size is odd). The files are `video-[-].mp4` and + /// `frames-[-].csv` in `dir`. + /// + /// # Errors + /// + /// Returns an error when: + /// + /// - `config.codec` is [`Codec::Hevc`]: only H.264 is recorded on Windows; + /// - Windows Runtime or Media Foundation cannot be started; + /// - `display` is no longer attached (`display is not available for capture (yet)`), or + /// Windows would not capture it; + /// - Windows.Graphics.Capture is not available (Windows 10 version 1803 or later is needed); + /// - Direct3D 11 cannot create a hardware device, or the device cannot be given to the capture + /// or the encoder; + /// - the video file cannot be created, the H.264 encoder does not accept the size, or the + /// encoder cannot take frames of the input format, or writing cannot begin; + /// - the frame log cannot be created in `dir`; + /// - the capture's frame pool or session cannot be created, or Windows refuses to start the + /// capture. + /// + /// A failure after the sink writer was created leaves the video file behind. A caller that + /// retries gets the same segment number again ([`CaptureBackend::start`]), and so the same file + /// name. + pub fn start( + display: &DisplayDescriptor, + segment: u32, + config: &CaptureConfig, + dir: &Path, + ) -> eyre::Result { + if config.codec == Codec::Hevc { + eyre::bail!( + "HEVC cannot be recorded on Windows yet (Media Foundation's MP4 writer does not \ + write it); use --codec h264" + ); + } + + crate::platform::media::start()?; + let item = item_for(display)?; + let size = item.Size()?; + let (width, height) = ( + u32::try_from(even(f64::from(size.Width)))?, + u32::try_from(even(f64::from(size.Height)))?, + ); + + let gpu = Gpu::new()?; + let file = video_file_name(display.id, segment); + let frames_file = frames_file_name(display.id, segment); + let path = dir.join(&file); + let writer = Writer::create(&path, &gpu, width, height, config.fps)?; + let log = FrameLog::create(&dir.join(&frames_file)) + .wrap_err_with(|| format!("could not create {frames_file}"))?; + + let shared = Arc::new(Shared { + stats: Arc::new(CaptureStats::default()), + state: Mutex::new(State { + writer, + placement: Placement::default(), + log, + latency: Latency::default(), + last_frame: None, + started: false, + finished: false, + }), + gpu, + width, + height, + source: size, + }); + + let pool = Direct3D11CaptureFramePool::CreateFreeThreaded( + &shared.gpu.winrt, + DirectXPixelFormat::B8G8R8A8UIntNormalized, + POOL_FRAMES, + SizeInt32 { + Width: size.Width, + Height: size.Height, + }, + ) + .wrap_err("could not create the capture's frame pool")?; + + let on_arrived = shared.clone(); + pool.FrameArrived(&TypedEventHandler::new( + move |pool: windows::core::Ref, _| { + if let Some(pool) = pool.as_ref() { + on_frame(&on_arrived, pool); + } + Ok(()) + }, + ))?; + + // the display going away closes its item + let on_closed = shared.clone(); + item.Closed(&TypedEventHandler::new(move |_, _| { + on_closed.stats.set_error( + "the display was closed (removed, or its mode changed)".to_string(), + clock::mono_now_ns(), + ); + Ok(()) + }))?; + + let session = pool + .CreateCaptureSession(&item) + .wrap_err("could not begin a capture session")?; + // best effort: each of these needs a newer Windows than the capture itself + let _ = session.SetIsBorderRequired(false); + let _ = session.SetIsCursorCaptureEnabled(true); + session + .StartCapture() + .wrap_err("Windows refused to start the capture")?; + + Ok(Self { + descriptor: display.clone(), + segment, + width_px: width as usize, + height_px: height as usize, + codec: Codec::H264, + file, + frames_file, + path, + _item: item, + pool, + session, + shared, + }) + } + + fn result( + &self, + reason: Option, + end_ns: Option, + error: Option, + ) -> DisplayResult { + let state = self.shared.state.lock(); + let stats = &self.shared.stats; + DisplayResult { + display: self.descriptor.clone(), + segment: self.segment, + // frames are native pixels + scale: 1.0, + width_px: self.width_px, + height_px: self.height_px, + codec: self.codec, + file: self.file.clone(), + frames_file: self.frames_file.clone(), + video_t0_mach_ns: stats.first_pts_mach_ns(), + video_end_mach_ns: end_ns, + end_reason: reason, + frames: state.log.frames(), + dropped: stats.dropped.load(Ordering::Relaxed), + // Windows.Graphics.Capture does not call back for a frame with nothing new + idle_callbacks: 0, + latency: state.latency.finish(), + error, + } + } +} + +impl RunningCapture for DisplayCapture { + fn display_id(&self) -> u32 { + self.descriptor.id + } + + fn segment(&self) -> u32 { + self.segment + } + + fn stats(&self) -> Arc { + self.shared.stats.clone() + } + + fn partial(&self) -> DisplayResult { + self.result(None, None, self.shared.stats.error()) + } + + /// Windows.Graphics.Capture says nothing while the picture does not change, so silence is not a + /// sign of death here and a stalled stream cannot be told by it. A display that went away is + /// told by its item closing, a GPU that went away by its device, and either by the display + /// poll. + fn last_signal_mach_ns(&self) -> Option { + None + } + + fn keepalive(&self, now_ns: u64) { + if let Some(reason) = self.shared.gpu.removed() { + self.shared.stats.set_error(reason, now_ns); + self.shared.state.lock().finished = true; + return; + } + + let mut state = self.shared.state.lock(); + if !state.started || state.finished { + return; + } + if !keepalive_due(now_ns, state.placement.last_ns(), None) { + return; + } + + let Some(last) = state.last_frame.clone() else { + return; + }; + let Some(placed) = state.placement.repeat(now_ns) else { + return; + }; + + if let Err(error) = state.writer.write(&last, placed.file_time) { + self.shared.stats.set_error(format!("{error:#}"), now_ns); + state.finished = true; + return; + } + + // not a new frame, and marked so + state.log.repeat(placed.pts_ns, now_ns); + } + + fn stop(self: Box, reason: EndReason, end_mach_ns: Option) -> DisplayResult { + // no more frames: ends the session, then the pool, then waits out a callback in flight by + // taking the lock below + let _ = self.session.Close(); + let _ = self.pool.Close(); + + let mut error = None; + let end_ns = { + let mut state = self.shared.state.lock(); + state.finished = true; + state.log.flush(); + + // A still display produced no frames, so without an explicit end the file would stop + // at the last change and late timecodes would fall off it. A segment that ended early + // (a dead capture) ends when it died, never before its last frame. + let now = clock::mono_now_ns(); + let end_ns = end_mach_ns + .unwrap_or(now) + .min(now) + .max(state.placement.last_ns()); + + if state.started { + if let Some(last) = state.last_frame.clone() + && end_ns > state.placement.last_ns() + && let Some(placed) = state.placement.repeat(end_ns) + { + match state.writer.write(&last, placed.file_time) { + Ok(()) => state.log.repeat(placed.pts_ns, now), + Err(write_error) => error = Some(format!("{write_error:#}")), + } + state.log.flush(); + } + + if let Err(finish_error) = state.writer.finish() { + error.get_or_insert_with(|| format!("{finish_error:#}")); + } + } else { + // never began: there is no video, and the empty file is no use to anyone + drop(state); + let _ = std::fs::remove_file(&self.path); + error = Some("no frames were captured".to_string()); + return self.result( + Some(reason), + Some(end_ns), + error.or_else(|| self.shared.stats.error()), + ); + } + + end_ns + }; + + let error = error.or_else(|| self.shared.stats.error()); + self.result(Some(reason), Some(end_ns), error) + } +} + +/// The real Windows [`CaptureBackend`]. +/// +/// [`active_displays`](CaptureBackend::active_displays) is +/// [`crate::platform::capture::active_displays`], which lists the monitors from the window system +/// and needs no permission, and [`start`](CaptureBackend::start) starts a [`DisplayCapture`]. On +/// Windows, [`crate::platform::capture::Backend`] is this type. +pub struct WgcBackend; + +impl CaptureBackend for WgcBackend { + fn active_displays(&self) -> eyre::Result> { + crate::platform::capture::active_displays() + } + + fn start( + &self, + display: &DisplayDescriptor, + segment: u32, + config: &CaptureConfig, + dir: &Path, + ) -> eyre::Result> { + Ok(Box::new(DisplayCapture::start( + display, segment, config, dir, + )?)) + } +} diff --git a/komorebi-recorder/src/lib.rs b/komorebi-recorder/src/lib.rs index b6fed000..f042f9ba 100644 --- a/komorebi-recorder/src/lib.rs +++ b/komorebi-recorder/src/lib.rs @@ -48,6 +48,7 @@ //! | [`lock`] | telling a running recorder from one that died | //! | [`sources`] | the sources that write raw files into the session | //! | [`session`] | the session directory and its manifest | +//! | [`capture`] | capturing the screen into segments of video | //! //! # What is not supported //! @@ -61,6 +62,7 @@ //! [`platform`] for what each host provides. pub mod analysis; +pub mod capture; pub mod clock; pub mod incidents; pub mod komorebi_link; diff --git a/komorebi-recorder/src/platform/macos/capture.rs b/komorebi-recorder/src/platform/macos/capture.rs new file mode 100644 index 00000000..598080d2 --- /dev/null +++ b/komorebi-recorder/src/platform/macos/capture.rs @@ -0,0 +1,67 @@ +//! What this host can capture: the displays, the permission to capture them, the capture backend, +//! and the duration of a recorded video. +//! +//! Screen capture on macOS is `ScreenCaptureKit` and `AVFoundation`; the code lives in +//! [`crate::capture::sck`] and this is the host-neutral name for it. +//! +//! | item | on macOS | +//! |---|---| +//! | [`supported`] | always `true` | +//! | [`has_permission`] | whether Screen Recording permission is granted; never prompts | +//! | [`ensure_permission`] | prompts for the permission if it is not granted, and fails until it is | +//! | [`active_displays`] | the displays CoreGraphics lists as active, in points | +//! | [`main_display_id`], [`main_display`] | the main display | +//! | [`media_duration_s`] | the duration of a video file, from `AVFoundation` | +//! | [`Backend`] | [`SckBackend`](crate::capture::sck::SckBackend), the [`CaptureBackend`](crate::capture::CaptureBackend) that records with `ScreenCaptureKit` | +//! +//! The same items exist on Windows. A display's `frame` is in points on a Mac and in pixels on +//! Windows ([`DisplayDescriptor::scale`] converts), and Windows needs no permission. + +use crate::capture::DisplayDescriptor; +use color_eyre::eyre; +use objc2_core_graphics::CGMainDisplayID; + +/// The [`CaptureBackend`](crate::capture::CaptureBackend) of this host: `ScreenCaptureKit`. +pub use crate::capture::sck::SckBackend as Backend; +/// Returns the displays attached right now, from CoreGraphics, with frames in points. It fails +/// only if CoreGraphics cannot list them; see [`crate::capture::sck::active_displays`]. +pub use crate::capture::sck::active_displays; +/// Makes sure this process may capture the screen, prompting for the Screen Recording permission +/// if it must, and fails until it is granted; see [`crate::capture::sck::ensure_permission`]. +pub use crate::capture::sck::ensure_permission; +/// Returns `true` if and only if the Screen Recording permission is already granted. It never +/// prompts; see [`crate::capture::sck::has_permission`]. +pub use crate::capture::sck::has_permission; +/// Returns the duration of a video file in seconds as `AVFoundation` reads it, or `None` if it +/// cannot be read; see [`crate::capture::sck::media_duration_s`]. +pub use crate::capture::sck::media_duration_s; + +/// Returns `true` if and only if this host can capture the screen. It is always `true` on macOS +/// (and on Windows), and `false` on a host that can only investigate a session. +#[must_use] +pub fn supported() -> bool { + true +} + +/// Returns the id of the display the OS treats as the main one (`CGMainDisplayID`). +/// +/// This is the same id as [`DisplayDescriptor::id`] of that display in [`active_displays`]. On +/// Windows it is the primary display's, or `0` if there are none. +#[must_use] +pub fn main_display_id() -> u32 { + CGMainDisplayID() +} + +/// Returns the main display among the currently attached ones. +/// +/// # Errors +/// +/// Returns an error if the displays cannot be listed (see [`active_displays`]), or if the main +/// display is not among them (the displays can change between the two reads). +pub fn main_display() -> eyre::Result { + let main = main_display_id(); + active_displays()? + .into_iter() + .find(|display| display.id == main) + .ok_or_else(|| eyre::Report::msg("the main display is not available for capture")) +} diff --git a/komorebi-recorder/src/platform/macos/mod.rs b/komorebi-recorder/src/platform/macos/mod.rs index eb24c48b..af1b6d0e 100644 --- a/komorebi-recorder/src/platform/macos/mod.rs +++ b/komorebi-recorder/src/platform/macos/mod.rs @@ -15,6 +15,7 @@ //! re-exports its submodules under the same names on every host. pub mod apps; +pub mod capture; pub mod clock; pub mod process; pub mod shell; diff --git a/komorebi-recorder/src/platform/mod.rs b/komorebi-recorder/src/platform/mod.rs index 852352cf..3cc62b46 100644 --- a/komorebi-recorder/src/platform/mod.rs +++ b/komorebi-recorder/src/platform/mod.rs @@ -10,6 +10,7 @@ //! | [`apps`] | which process owns a window id | //! | [`process`] | stopping a child process | //! | [`trace`] | streaming the OS trace into the session | +//! | [`capture`] | can we capture, which displays exist, the real [`CaptureBackend`], a video's duration | //! //! A third host, `unsupported`, builds on anything else so that the investigation commands (which //! only read a session directory) still work on a Linux agent box; recording there says so. @@ -28,6 +29,8 @@ //! compiler enforces that for the host being built, and for no other: the Windows host is built //! from a Mac by `just check-windows` and by CI, and `unsupported` is built only on a third OS, //! where nothing in CI builds it. +//! +//! [`CaptureBackend`]: crate::capture::CaptureBackend #[cfg(target_os = "macos")] mod macos; @@ -46,7 +49,10 @@ mod win; use win as imp; pub use imp::apps; +pub use imp::capture; pub use imp::clock; +#[cfg(windows)] +pub use imp::media; pub use imp::process; pub use imp::shell; pub use imp::trace; diff --git a/komorebi-recorder/src/platform/unsupported.rs b/komorebi-recorder/src/platform/unsupported.rs index d9a0c064..c047084e 100644 --- a/komorebi-recorder/src/platform/unsupported.rs +++ b/komorebi-recorder/src/platform/unsupported.rs @@ -16,6 +16,93 @@ pub mod apps { } } +/// Screen capture, which this host cannot do. +pub mod capture { + use crate::capture::CaptureBackend; + use crate::capture::CaptureConfig; + use crate::capture::DisplayDescriptor; + use crate::capture::RunningCapture; + use color_eyre::eyre; + use std::path::Path; + + fn unsupported() -> eyre::Result { + eyre::bail!( + "recording the screen is not supported on {}; investigating a session recorded \ + elsewhere is", + std::env::consts::OS + ) + } + + /// Returns `false`: there is no screen to capture. + pub fn has_permission() -> bool { + false + } + + /// Returns `false`: this host cannot capture the screen. + #[must_use] + pub fn supported() -> bool { + false + } + + /// Fails: this host cannot capture the screen. + /// + /// # Errors + /// + /// Always returns an error that says recording is not supported here. + pub fn ensure_permission() -> eyre::Result<()> { + unsupported() + } + + /// Fails: this host has no displays it can capture. + /// + /// # Errors + /// + /// Always returns an error that says recording is not supported here. + pub fn active_displays() -> eyre::Result> { + unsupported() + } + + /// Returns `0`: there is no main display. + #[must_use] + pub fn main_display_id() -> u32 { + 0 + } + + /// Fails: this host has no main display. + /// + /// # Errors + /// + /// Always returns an error that says recording is not supported here. + pub fn main_display() -> eyre::Result { + unsupported() + } + + /// Returns `None`: this host has nothing to read a video's duration with. + #[must_use] + pub fn media_duration_s(_path: &Path) -> Option { + None + } + + /// A capture backend that cannot capture. + pub struct Backend; + + impl CaptureBackend for Backend { + fn active_displays(&self) -> eyre::Result> { + active_displays() + } + + fn start( + &self, + _display: &DisplayDescriptor, + _segment: u32, + _config: &CaptureConfig, + _dir: &Path, + ) -> eyre::Result> { + unsupported() + } + } +} + /// The master clock, which here is Rust's monotonic [`Instant`]. pub mod clock { use crate::clock::Timebase; diff --git a/komorebi-recorder/src/platform/win/capture.rs b/komorebi-recorder/src/platform/win/capture.rs new file mode 100644 index 00000000..099563dc --- /dev/null +++ b/komorebi-recorder/src/platform/win/capture.rs @@ -0,0 +1,226 @@ +//! What this host can capture: the displays, the permission to capture them, the capture backend, +//! and the duration of a recorded video. +//! +//! Screen capture on Windows is Windows.Graphics.Capture into a Media Foundation writer; the code +//! lives in [`crate::capture::wgc`] and [`Backend`] is the host-neutral name for it. What this +//! module itself does is list the monitors and read their scale. +//! +//! | item | on Windows | +//! |---|---| +//! | [`supported`] | always `true` | +//! | [`has_permission`] | always `true`: there is nothing to grant | +//! | [`ensure_permission`] | always succeeds | +//! | [`active_displays`] | the monitors `EnumDisplayMonitors` lists, in physical pixels | +//! | [`main_display_id`], [`main_display`] | the primary monitor | +//! | [`media_duration_s`] | the duration of a video file, from Media Foundation | +//! | [`Backend`] | [`WgcBackend`](crate::capture::wgc::WgcBackend), the [`CaptureBackend`](crate::capture::CaptureBackend) that records with Windows.Graphics.Capture | +//! +//! The same items exist on macOS, which needs the Screen Recording permission and reports displays +//! in points. Windows-only here: [`become_per_monitor_dpi_aware`] and [`monitor_handle`]. + +use crate::capture::DisplayDescriptor; +use color_eyre::eyre; +use std::path::Path; +use windows::Win32::Foundation::LPARAM; +use windows::Win32::Foundation::RECT; +use windows::Win32::Graphics::Gdi::EnumDisplayMonitors; +use windows::Win32::Graphics::Gdi::GetMonitorInfoW; +use windows::Win32::Graphics::Gdi::HDC; +use windows::Win32::Graphics::Gdi::HMONITOR; +use windows::Win32::Graphics::Gdi::MONITORINFO; +use windows::Win32::UI::HiDpi::DPI_AWARENESS_CONTEXT_PER_MONITOR_AWARE_V2; +use windows::Win32::UI::HiDpi::GetDpiForMonitor; +use windows::Win32::UI::HiDpi::MDT_EFFECTIVE_DPI; +use windows::Win32::UI::HiDpi::SetProcessDpiAwarenessContext; +use windows::core::BOOL; + +/// `MONITORINFO::dwFlags` for the primary monitor. +const MONITORINFOF_PRIMARY: u32 = 1; + +/// Windows reports 96 dpi for a display at 100 %. +const BASE_DPI: f64 = 96.0; + +/// Makes this process per-monitor DPI aware (v2), so that every size and position it reads is in +/// physical pixels, the same as komorebi's. Fails harmlessly when the process already is. +/// +/// This must run before the process creates a window or reads a size: `platform::init` calls it +/// first thing in `main`, on Windows. +pub fn become_per_monitor_dpi_aware() { + // SAFETY: no pointers involved + let _ = unsafe { SetProcessDpiAwarenessContext(DPI_AWARENESS_CONTEXT_PER_MONITOR_AWARE_V2) }; +} + +/// Returns `true` if and only if this host can capture the screen. It is always `true` on Windows +/// (and on macOS), and `false` on a host that can only investigate a session. +#[must_use] +pub fn supported() -> bool { + true +} + +/// Returns `true` if and only if this process may capture the screen. It always may. +/// +/// There is nothing to grant, because Windows lets a process capture the screen without asking. +/// macOS needs the Screen Recording permission. +#[must_use] +pub fn has_permission() -> bool { + true +} + +/// Makes sure this process may capture the screen, asking the user if it must. On Windows it +/// needs nothing, so this always succeeds and asks nothing. +/// +/// # Errors +/// +/// Never fails on this host. (On macOS it fails until the Screen Recording permission is granted.) +pub fn ensure_permission() -> eyre::Result<()> { + Ok(()) +} + +struct Found { + monitor: HMONITOR, + /// Left, top, right, bottom in virtual-screen pixels. + rect: [i32; 4], + primary: bool, +} + +unsafe extern "system" fn collect( + monitor: HMONITOR, + _context: HDC, + _rect: *mut RECT, + found: LPARAM, +) -> BOOL { + // SAFETY: `found` is the address of the vector `monitors` passed to `EnumDisplayMonitors` + let found = unsafe { &mut *(found.0 as *mut Vec) }; + let mut info = MONITORINFO { + cbSize: size_of::() as u32, + ..Default::default() + }; + // SAFETY: `info` is a valid, correctly sized out structure + if unsafe { GetMonitorInfoW(monitor, &raw mut info) }.as_bool() { + found.push(Found { + monitor, + rect: [ + info.rcMonitor.left, + info.rcMonitor.top, + info.rcMonitor.right, + info.rcMonitor.bottom, + ], + primary: info.dwFlags & MONITORINFOF_PRIMARY != 0, + }); + } + + BOOL(1) +} + +fn monitors() -> Vec { + let mut found: Vec = Vec::new(); + // SAFETY: `found` outlives the call and `collect` only touches it during it + let _ = + unsafe { EnumDisplayMonitors(None, None, Some(collect), LPARAM(&raw mut found as isize)) }; + found +} + +/// Returns the monitor with this id, if it is still attached. The id is the one +/// [`active_displays`] gives a display: the monitor's `HMONITOR`. +/// +/// Windows only: the capture backend needs the handle to create a capture item for a display. +#[must_use] +pub fn monitor_handle(id: u32) -> Option { + monitors() + .into_iter() + .map(|found| found.monitor) + .find(|monitor| id_of(*monitor) == id) +} + +/// A monitor's id: its `HMONITOR`, which is the id komorebi for Windows gives it too. Handles are +/// 32-bit values even in a 64-bit process. +fn id_of(monitor: HMONITOR) -> u32 { + monitor.0 as usize as u32 +} + +fn scale_of(monitor: HMONITOR) -> f64 { + let (mut x, mut y) = (0_u32, 0_u32); + // SAFETY: both are valid out pointers for the duration of the call + let known = unsafe { GetDpiForMonitor(monitor, MDT_EFFECTIVE_DPI, &raw mut x, &raw mut y) }; + if known.is_ok() && x > 0 { + f64::from(x) / BASE_DPI + } else { + 1.0 + } +} + +/// Returns the displays attached right now. +/// +/// A display's `id` is its `HMONITOR` as a `u32`, which is the id komorebi for Windows gives it +/// too. Its `frame` is `[left, top, width, height]` in the virtual screen's physical pixels, so +/// `width_points` and `height_points` are its width and height in pixels and `scale` is the +/// monitor's effective DPI over 96 (`1.0` if Windows will not say). `asleep` is always `false`. On +/// macOS the frame is in points and a sleeping display is reported as asleep. +/// +/// # Errors +/// +/// Never fails on this host: a monitor list that cannot be read is an empty list. (On macOS it +/// fails if CoreGraphics cannot list the displays.) +pub fn active_displays() -> eyre::Result> { + Ok(monitors() + .into_iter() + .map(|found| { + let [left, top, right, bottom] = found.rect; + let (width, height) = (right - left, bottom - top); + DisplayDescriptor { + id: id_of(found.monitor), + frame: [ + f64::from(left), + f64::from(top), + f64::from(width), + f64::from(height), + ], + width_points: i64::from(width), + height_points: i64::from(height), + asleep: false, + scale: scale_of(found.monitor), + } + }) + .collect()) +} + +/// Returns the id of the primary display (the same id as [`DisplayDescriptor::id`] of that display +/// in [`active_displays`]), or `0` if there are none. +#[must_use] +pub fn main_display_id() -> u32 { + monitors() + .iter() + .find(|found| found.primary) + .map_or(0, |found| id_of(found.monitor)) +} + +/// Returns the primary display. +/// +/// # Errors +/// +/// Returns an error if the primary display is not among the displays attached, which includes +/// there being none. +pub fn main_display() -> eyre::Result { + let main = main_display_id(); + active_displays()? + .into_iter() + .find(|display| display.id == main) + .ok_or_else(|| eyre::Report::msg("the primary display is not available for capture")) +} + +/// Returns the playable duration of a video file as Media Foundation reads it, in seconds. +/// +/// This is what recovery trusts for a file whose recorder died: the frame log is flushed on a +/// timer and can lag behind (or run ahead of) what actually reached the video. A file whose +/// duration it cannot read (some truncated ones), or that is not longer than zero, is `None`, and +/// recovery says it could not tell. On macOS the duration comes from `AVFoundation`. +/// +/// Media Foundation's reader is not part of the build yet, so this is always `None`. +#[must_use] +pub fn media_duration_s(path: &Path) -> Option { + let _ = path; + None +} + +/// The [`CaptureBackend`](crate::capture::CaptureBackend) of this host: Windows.Graphics.Capture. +pub use crate::capture::wgc::WgcBackend as Backend; diff --git a/komorebi-recorder/src/platform/win/media.rs b/komorebi-recorder/src/platform/win/media.rs new file mode 100644 index 00000000..c161885f --- /dev/null +++ b/komorebi-recorder/src/platform/win/media.rs @@ -0,0 +1,64 @@ +//! Starting Media Foundation, and packing the values it keeps in one 64-bit attribute. +//! +//! The code that writes video (`crate::capture::wgc`) and the code that reads it back +//! (`crate::av`) both need Windows Runtime and Media Foundation running: [`start`] does that once +//! for the process. [`pack`] and [`unpack`] build and take apart the attributes that hold a size +//! or a ratio. +//! +//! This module exists only on Windows. It is reached as `crate::platform::media`, and the facade +//! has no counterpart on other hosts: nothing there writes or reads video through Media +//! Foundation. Nothing tests it automatically; it is exercised by recording and by playing back +//! video on Windows. + +use color_eyre::eyre; +use std::sync::OnceLock; +use windows::Win32::Media::MediaFoundation::MF_VERSION; +use windows::Win32::Media::MediaFoundation::MFSTARTUP_FULL; +use windows::Win32::Media::MediaFoundation::MFStartup; +use windows::Win32::System::WinRT::RO_INIT_MULTITHREADED; +use windows::Win32::System::WinRT::RoInitialize; + +/// Starts Windows Runtime and Media Foundation for this process, once. Never shut down: the process +/// is short-lived and a restarted capture or decoder would only start it again. +/// +/// The first call does the work and every later call returns the same outcome, so callers need not +/// coordinate. A thread that Windows Runtime was already initialized on, or in a different +/// apartment, is fine. +/// +/// # Errors +/// +/// Returns an error if Media Foundation cannot be started (`MFStartup` fails), with Windows' +/// reason. The failure is kept: later calls fail the same way without trying again. +pub fn start() -> eyre::Result<()> { + static STARTED: OnceLock> = OnceLock::new(); + STARTED + .get_or_init(|| { + // SAFETY: plain initialisation calls; a thread that is already initialised, or in a + // different apartment, is fine and says so with an error that is ignored + unsafe { + let _ = RoInitialize(RO_INIT_MULTITHREADED); + MFStartup(MF_VERSION, MFSTARTUP_FULL).map_err(|error| error.to_string()) + } + }) + .clone() + .map_err(|error| eyre::Report::msg(format!("could not start Media Foundation: {error}"))) +} + +/// Packs two 32-bit values into the one 64-bit attribute Media Foundation uses for a size or a +/// ratio. +/// +/// `high` goes in the upper 32 bits and `low` in the lower. The inline helpers `MFSetAttributeSize` +/// and `MFSetAttributeRatio` do this, and are not bound. A frame size is `pack(width, height)` and +/// a ratio `pack(numerator, denominator)`. +/// +/// [`unpack`] undoes it: `unpack(pack(a, b)) == (a, b)` for every `a` and `b`. +#[must_use] +pub const fn pack(high: u32, low: u32) -> u64 { + ((high as u64) << 32) | low as u64 +} + +/// Returns the two halves of such an attribute, `(high, low)`: the inverse of [`pack`]. +#[must_use] +pub const fn unpack(value: u64) -> (u32, u32) { + ((value >> 32) as u32, value as u32) +} diff --git a/komorebi-recorder/src/platform/win/mod.rs b/komorebi-recorder/src/platform/win/mod.rs index c0287a83..cfc9566d 100644 --- a/komorebi-recorder/src/platform/win/mod.rs +++ b/komorebi-recorder/src/platform/win/mod.rs @@ -22,7 +22,9 @@ //! `docs/dev/windows-checkpoints.md`. pub mod apps; +pub mod capture; pub mod clock; +pub mod media; pub mod process; pub mod shell; pub mod trace; @@ -33,4 +35,5 @@ pub mod trace; pub fn init() { // Without this Windows lies to us about sizes: a display scaled to 150 % reads as smaller than // it is, and komorebi's rects (which are physical pixels) would not match ours. + capture::become_per_monitor_dpi_aware(); } diff --git a/komorebi-recorder/src/session.rs b/komorebi-recorder/src/session.rs index b097042b..643dbf5f 100644 --- a/komorebi-recorder/src/session.rs +++ b/komorebi-recorder/src/session.rs @@ -4,6 +4,7 @@ //! the raw files, so streams stay separate and can be reconciled or re-indexed later. use crate::analysis::flavor::Flavor; +use crate::capture::DisplayResult; use crate::clock::Anchor; use crate::clock::Timebase; use crate::platform::Host; @@ -101,6 +102,20 @@ pub struct KomorebiInfo { pub log_dir: Option, } +/// A display's capture plus where its file sits relative to the session's video time. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct DisplayEntry { + #[serde(flatten)] + /// What capturing this segment produced. + pub result: DisplayResult, + /// Seconds between the session's video t=0 and this file's first frame (>= 0). A timestamp + /// `t` in session video time is at `t - offset_s` in this segment's file. + pub offset_s: f64, + /// Session video time at which this segment ended, once it has. + #[serde(default)] + pub end_s: Option, +} + /// The outcome of the self-test that checks the video against the clocks. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct Calibration { @@ -187,6 +202,8 @@ pub struct Manifest { pub video_t0_mach_ns: Option, /// When video capture was stopped on the mach clock. pub video_end_mach_ns: Option, + /// Every segment of every display's video, with its place in video time. + pub displays: Vec, /// The byte ranges of komorebi's log that were recorded, which together make `komorebi.log`. pub log_segments: Vec, /// The OS trace streams that were recorded, one per file under `oslog/`. @@ -225,6 +242,7 @@ impl Manifest { stop: None, video_t0_mach_ns: None, video_end_mach_ns: None, + displays: Vec::new(), log_segments: Vec::new(), oslog: Vec::new(), apps: Vec::new(), @@ -243,6 +261,67 @@ impl Manifest { .unwrap_or_else(|| Host::legacy_macos(self.macos_version.clone())) } + /// Install the display segments: every segment's offset from the session's video t=0 (the + /// earliest first frame of all of them) and the session-level video start and end. + pub fn apply_segments(&mut self, results: Vec) { + self.video_t0_mach_ns = results.iter().filter_map(|r| r.video_t0_mach_ns).min(); + let all_ended = + !results.is_empty() && results.iter().all(|r| r.video_end_mach_ns.is_some()); + self.video_end_mach_ns = if all_ended { + results.iter().filter_map(|r| r.video_end_mach_ns).max() + } else { + None + }; + + let t0 = self.video_t0_mach_ns; + self.displays = results + .into_iter() + .map(|result| { + let offset_s = match (result.video_t0_mach_ns, t0) { + (Some(this), Some(first)) => this.saturating_sub(first) as f64 / 1e9, + _ => 0.0, + }; + let end_s = result.video_end_mach_ns.map(|end| self.video_seconds(end)); + DisplayEntry { + result, + offset_s, + end_s, + } + }) + .collect(); + } + + /// The segment of `display` that has video at session video time `video_s`, if any. When a + /// display was unplugged or the machine slept there is a gap with no file to look in. + #[must_use] + pub fn segment_at(&self, display: u32, video_s: f64) -> Option<&DisplayEntry> { + // segment boundaries are sampled, not exact, so allow a frame either side + const SLACK_S: f64 = 0.05; + let (_, session_end) = self.video_span(); + + self.displays + .iter() + .filter(|entry| entry.result.display.id == display && entry.result.frames > 0) + .filter(|entry| { + video_s >= entry.offset_s - SLACK_S + && video_s <= entry.end_s.unwrap_or(session_end) + SLACK_S + }) + .max_by(|a, b| a.offset_s.total_cmp(&b.offset_s)) + } + + /// The distinct display ids that were recorded at any point. + #[must_use] + pub fn display_ids(&self) -> Vec { + let mut ids: Vec = self + .displays + .iter() + .map(|entry| entry.result.display.id) + .collect(); + ids.sort_unstable(); + ids.dedup(); + ids + } + /// The master clock value that is `t = 0` in video time. Falls back to the recording start if /// no frame was ever captured, so timelines remain usable for the non-video sources. #[must_use] @@ -719,6 +798,109 @@ mod tests { } } + fn segment(display: u32, number: u32, t0: u64, end: Option) -> DisplayResult { + DisplayResult { + display: crate::capture::DisplayDescriptor { + id: display, + frame: [0.0, 0.0, 100.0, 100.0], + width_points: 100, + height_points: 100, + asleep: false, + scale: 1.0, + }, + segment: number, + scale: 1.0, + width_px: 100, + height_px: 100, + codec: crate::capture::Codec::Hevc, + file: crate::capture::video_file_name(display, number), + frames_file: crate::capture::frames_file_name(display, number), + video_t0_mach_ns: Some(t0), + video_end_mach_ns: end, + end_reason: end.map(|_| crate::capture::EndReason::Stopped), + frames: 10, + dropped: 0, + idle_callbacks: 0, + latency: crate::capture::LatencyStats::default(), + error: None, + } + } + + const S: u64 = 1_000_000_000; + + /// Display 1 is recorded for 0-10s, unplugged, and back for 30-40s; display 2 for 2-40s. + fn docked_and_undocked() -> Manifest { + let mut manifest = manifest(); + manifest.apply_segments(vec![ + segment(1, 1, 100 * S, Some(110 * S)), + segment(2, 1, 102 * S, Some(140 * S)), + segment(1, 2, 130 * S, Some(140 * S)), + ]); + manifest.video_end_mach_ns = Some(140 * S); + manifest + } + + #[test] + fn segments_are_placed_relative_to_the_first_frame_of_any_of_them() { + let manifest = docked_and_undocked(); + assert_eq!(manifest.video_t0_mach_ns, Some(100 * S)); + + let by_file = |file: &str| { + manifest + .displays + .iter() + .find(|entry| entry.result.file == file) + .expect("segment") + }; + assert!((by_file("video-1.mp4").offset_s - 0.0).abs() < 1e-9); + assert!((by_file("video-2.mp4").offset_s - 2.0).abs() < 1e-9); + assert!((by_file("video-1-2.mp4").offset_s - 30.0).abs() < 1e-9); + assert_eq!(by_file("video-1.mp4").end_s, Some(10.0)); + assert_eq!(by_file("video-1-2.mp4").end_s, Some(40.0)); + assert_eq!(manifest.display_ids(), vec![1, 2]); + assert_eq!(manifest.video_span(), (0.0, 40.0)); + } + + #[test] + fn the_right_segment_is_found_and_a_gap_has_none() { + let manifest = docked_and_undocked(); + + let at = |display: u32, t: f64| { + manifest + .segment_at(display, t) + .map(|entry| entry.result.file.as_str()) + }; + + assert_eq!(at(1, 5.0), Some("video-1.mp4")); + assert_eq!(at(1, 35.0), Some("video-1-2.mp4")); + assert_eq!( + at(1, 20.0), + None, + "display 1 was unplugged between 10s and 30s" + ); + assert_eq!(at(2, 20.0), Some("video-2.mp4"), "display 2 never stopped"); + assert_eq!(at(2, 1.0), None, "display 2 had not started yet"); + assert_eq!(at(9, 5.0), None, "no such display"); + + // a frame of slack at each boundary absorbs sampling error + assert_eq!(at(1, 10.03), Some("video-1.mp4")); + assert_eq!(at(1, 10.2), None); + } + + #[test] + fn a_segment_still_being_recorded_extends_to_the_end_of_the_session() { + let mut manifest = manifest(); + manifest.apply_segments(vec![segment(1, 1, 100 * S, None)]); + manifest.stop = Some(Moment { + mach_ns: 160 * S, + wall_us: 0, + }); + manifest.video_end_mach_ns = Some(160 * S); + + assert!(manifest.segment_at(1, 50.0).is_some()); + assert!(manifest.segment_at(1, 59.9).is_some()); + } + #[test] fn manifests_from_before_segments_and_status_existed_still_load() { // exactly the shape a pre-segment recorder wrote: no `status`, no segment fields