diff --git a/Cargo.lock b/Cargo.lock index 4de535bd..d9dd36bf 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1907,6 +1907,7 @@ checksum = "e2db585e1d738fc771bf08a151420d3ed193d9d895a36df7f6f8a9456b911ddc" name = "komorebi-recorder" version = "0.1.0" dependencies = [ + "ab_glyph", "block2 0.6.2", "chrono", "clap", @@ -1917,6 +1918,7 @@ dependencies = [ "dispatch2", "dunce", "eframe", + "epaint_default_fonts", "libc", "objc2 0.6.4", "objc2-av-foundation", diff --git a/docs/usage/investigating-a-recording.md b/docs/usage/investigating-a-recording.md index 43337218..cd90a1ec 100644 --- a/docs/usage/investigating-a-recording.md +++ b/docs/usage/investigating-a-recording.md @@ -45,6 +45,8 @@ and `event:12.end` when it finished. | `window ID\|TEXT` | One window's life: when it appeared, moved, was parked off screen, was really somewhere else, and which event did each | | `code event:N`, `code log:N` | The line of komorebi's source that wrote a log line, with the code around it | | `search TEXT` | Where a word appears: komorebi's log, events, incidents, annotations (and `--oslog` for the macOS trace) | +| `changes` | When and where on each display the picture changed, and which events fall inside | +| `sheet` | A contact sheet image of frames with times, events and the bug region marked | | `annotate`, `annotations` | Add, change and list bug regions, notes and findings | ### Who changed it diff --git a/komorebi-recorder/Cargo.toml b/komorebi-recorder/Cargo.toml index c6d9d594..6d1b4e56 100644 --- a/komorebi-recorder/Cargo.toml +++ b/komorebi-recorder/Cargo.toml @@ -6,6 +6,7 @@ edition.workspace = true repository.workspace = true [dependencies] +ab_glyph = "0.2" chrono = { workspace = true } clap = { workspace = true } color-eyre = { workspace = true } @@ -14,6 +15,7 @@ ctrlc = { version = "3", features = ["termination"] } dirs = { workspace = true } dunce = { workspace = true } eframe = { workspace = true } +epaint_default_fonts = "0.33" parking_lot = { workspace = true } png = "0.18" regex = "1" diff --git a/komorebi-recorder/src/analysis/changes.rs b/komorebi-recorder/src/analysis/changes.rs new file mode 100644 index 00000000..d7c8ff53 --- /dev/null +++ b/komorebi-recorder/src/analysis/changes.rs @@ -0,0 +1,444 @@ +//! When, and where on the screen, the picture changed. +//! +//! komorebi's events say when it thought something happened; the video says when the screen +//! actually did. Comparing consecutive frames finds the stretches where pixels changed, how much +//! of the screen they covered and the box around them, so an investigator can see whether a layout +//! change was instant or took half a second, whether a window jumped, and what else on screen was +//! going on. Where an event falls inside a stretch of change it is named. +//! +//! [`Analysis::changes`] is what the `changes` command prints and what the viewer scans in the +//! background ([`Analysis::changes_until`]) to draw the screen lane of its timeline. +//! [`Analysis::video_gaps`] reports where a display had no video to compare at all. +//! +//! This is about pixels, not about komorebi's state: a [`ChangeSpan`] reports that the picture +//! moved, not what moved. The differences between two komorebi states are in [`super::diff`]. +//! +//! # How a change is found +//! +//! Each display is decoded separately and every frame is compared with the one before it: +//! +//! - Frames are scaled to 240 pixels wide and reduced to brightness (Rec. 601 weights). +//! - A pixel has changed if its brightness moved by more than 24 (of 255). +//! - A frame is a change if the share of its pixels that changed is at least 0.05% divided by the +//! `sensitivity` asked for: with a sensitivity of 2, changes half as large are noticed, and with +//! 0.5 only changes twice as large. +//! - Changed frames less than 0.15 s apart belong to one [`ChangeSpan`]. +//! - A frame is compared with the one before it on the same display, across the start of a new +//! segment too (a display that came back after being unplugged or asleep), so a screen that +//! looks different after the break is a change at the boundary. A frame of another size than +//! the one before it (a display that changed resolution) is not compared with it. +//! +//! Frames come when the screen changes, not on a schedule, so a display whose video starts less +//! than 0.5 s after the range starts, or stops less than 0.5 s before it ends, is not reported as +//! having a [`Gap`]. +//! +//! Decoding needs a video decoder on the host: where there is none, the functions here that read +//! frames return an error saying so, and everything else in a session can still be read. + +use super::Analysis; +use super::envelope; +use super::video::DecodedFrame; +use super::video::SessionVideo; +use crate::av::Scale; +use color_eyre::eyre; +use schemars::JsonSchema; +use serde::Serialize; + +/// The schema name that identifies a [`ChangesData`] in an envelope (`changes/1`); listed by the +/// `schema` command. +pub const SCHEMA_CHANGES: &str = "changes/1"; + +/// Frames are compared at this width: enough to see a window move, cheap enough to scan minutes. +const COMPARE_WIDTH: u32 = 240; + +/// A pixel counts as changed if its brightness moved by more than this (out of 255). +const PIXEL_THRESHOLD: i32 = 24; + +/// A frame is a change if at least this share of its pixels changed, at sensitivity 1. +const SHARE_THRESHOLD: f64 = 0.0005; + +/// Frames of one change closer together than this are one change. +const MERGE_GAP_S: f64 = 0.15; + +/// A display whose first frame arrives this late, or whose video stops this early, is not +/// reported as having a gap: frames come when the screen changes, not on a schedule. +const MIN_GAP_S: f64 = 0.5; + +/// An event this close to a change is said to coincide with it. +const COINCIDE_S: f64 = 0.25; + +/// A stretch of video in which the picture of one display changed. +/// +/// A span of a single changed frame has `start_video_s == end_video_s`. +#[derive(Debug, Clone, PartialEq, Serialize, JsonSchema)] +pub struct ChangeSpan { + /// The id of the display, the same number as [`Monitor::id`](super::state::Monitor::id) and + /// the `--display` of the recording. + pub display: u32, + /// When the first changed frame was on screen, in seconds of video time. + pub start_video_s: f64, + /// When the last changed frame was on screen, in seconds of video time. + pub end_video_s: f64, + /// Changed frames in the span: not the frames that fall between its start and end. + pub frames: usize, + /// The largest share of the screen that changed between two consecutive frames. + pub peak_share: f64, + /// The box around everything that changed, as fractions of the screen: `[x, y, width, height]`. + /// The fractions are of the size of the last frame decoded for the display. + pub bbox: [f64; 4], + /// The same box in global display points, if the display's frame is known: + /// `[x, y, width, height]`. + pub bbox_points: Option<[f64; 4]>, + /// The `seq` of each komorebi event received from 0.25 s before the span starts to 0.25 s + /// after it ends. An event is received when its handler has finished. + pub events: Vec, +} + +/// A stretch of a range in which a display had no video at all. +/// +/// A display has none while it is unplugged or asleep, or after its capture failed. +#[derive(Debug, Clone, PartialEq, Serialize, JsonSchema)] +pub struct Gap { + /// The id of the display. + pub display: u32, + /// Where the gap starts, in seconds of video time. + pub from_video_s: f64, + /// Where the gap ends, in seconds of video time. + pub to_video_s: f64, +} + +/// Where the picture changed on each display over a range of the recording: the `data` of the +/// `changes` command. +#[derive(Debug, Clone, PartialEq, Serialize, JsonSchema)] +pub struct ChangesData { + /// The start of the range that was scanned, in seconds of video time. + pub from_video_s: f64, + /// The end of the range that was scanned, in seconds of video time. + pub to_video_s: f64, + /// The displays that were scanned, in the order asked for. + pub displays: Vec, + /// How many frames were decoded and looked at, over all displays. A scan that was cancelled + /// counts only what it got through. + pub frames_examined: usize, + /// The changes, ordered by start time and then by display. + pub spans: Vec, + /// Stretches of the range where a display had no video at all (unplugged, asleep). See + /// [`Analysis::video_gaps`]. + pub gaps: Vec, +} + +/// The brightness of every pixel of a BGRA frame. +fn luma(frame: &DecodedFrame) -> Vec { + frame + .frame + .bgra + .as_chunks::<4>() + .0 + .iter() + .map(|pixel| { + // Rec. 601 weights in integers + ((u32::from(pixel[2]) * 299 + u32::from(pixel[1]) * 587 + u32::from(pixel[0]) * 114) + / 1000) as u8 + }) + .collect() +} + +struct Open { + start: f64, + end: f64, + frames: usize, + peak: f64, + bounds: (u32, u32, u32, u32), +} + +impl Analysis { + /// Finds where the picture changed on each of `displays` in `from_s..=to_s` (seconds of video + /// time). + /// + /// `sensitivity` scales how little has to change for a frame to count: the default of the + /// `changes` command is `1.0`, a larger number notices smaller changes and a smaller one only + /// larger changes. How a change is found is described in the [module](self) documentation. + /// A display with no video in the range contributes no spans and is reported in + /// [`ChangesData::gaps`]; so is one that is not in the session at all. + /// + /// This decodes every frame of the range, so its cost is that of decoding the video, which was + /// measured at 250 to 970 frames per second on macOS (HEVC, depending on frame size). `video` + /// belongs to the thread that made it (a `SessionVideo` is not `Send`), so a scan on another + /// thread builds a new `SessionVideo`. + /// + /// # Errors + /// + /// Returns an error if: + /// + /// - `sensitivity` is not above zero (the usage exit kind); + /// - a display's video file cannot be resolved inside the session, opened or decoded (missing, + /// truncated or unreadable), or the host has no video decoder. + /// + /// # Example + /// + /// This needs recorded video, so it is not run here. It is covered by + /// `tests/video.rs::the_picture_changing_is_found_where_it_changed_and_not_where_it_held_still` + /// and `tests/video.rs::a_display_with_no_video_for_part_of_the_range_has_a_gap`, which build + /// sessions with numbered videos and run the `changes` command over them. + /// + /// ```rust,no_run + /// use komorebi_recorder::analysis::Analysis; + /// use komorebi_recorder::analysis::guard::SessionPath; + /// use komorebi_recorder::analysis::video::SessionVideo; + /// + /// let analysis = Analysis::open(std::path::Path::new("session-dir"))?; + /// let guard = SessionPath::new(&analysis.session.dir)?; + /// let video = SessionVideo::new(&analysis.manifest, &guard); + /// + /// let data = analysis.changes(&video, &video.displays(), 0.0, 60.0, 1.0)?; + /// for span in &data.spans { + /// println!( + /// "display {}: {:.2} s to {:.2} s, events {:?}", + /// span.display, span.start_video_s, span.end_video_s, span.events + /// ); + /// } + /// # Ok::<(), Box>(()) + /// ``` + pub fn changes( + &self, + video: &SessionVideo, + displays: &[u32], + from_s: f64, + to_s: f64, + sensitivity: f64, + ) -> eyre::Result { + self.changes_until(video, displays, from_s, to_s, sensitivity, None) + } + + /// Finds where the picture changed, like [`Analysis::changes`], and stops early with what it + /// has found so far once `cancel` is set: for a scan that runs in the background and whose + /// result is no longer wanted. + /// + /// `cancel` is checked once per decoded frame, so a scan stops within a frame of it being set. + /// A cancelled scan is not an error: it returns the spans of what it got through, and + /// [`ChangesData::frames_examined`] says how far that was. With `None` it runs to the end, + /// which is what [`Analysis::changes`] does. + /// + /// # Errors + /// + /// The same as [`Analysis::changes`]. + pub fn changes_until( + &self, + video: &SessionVideo, + displays: &[u32], + from_s: f64, + to_s: f64, + sensitivity: f64, + cancel: Option<&std::sync::atomic::AtomicBool>, + ) -> eyre::Result { + if sensitivity <= 0.0 { + return Err(envelope::usage("--sensitivity must be above zero")); + } + + let threshold = SHARE_THRESHOLD / sensitivity; + let mut spans = Vec::new(); + let mut examined = 0; + + for display in displays { + let frame_box = self + .manifest + .displays + .iter() + .find(|entry| entry.result.display.id == *display) + .map(|entry| entry.result.display.frame); + + let mut previous: Option<(Vec, u32, u32)> = None; + let mut open: Option = None; + let mut finished: Vec = Vec::new(); + + examined += video.scan( + *display, + from_s, + to_s, + Scale::Width(COMPARE_WIDTH), + &mut |decoded| { + if cancel + .is_some_and(|cancel| cancel.load(std::sync::atomic::Ordering::Relaxed)) + { + return false; + } + let now = luma(decoded); + let (width, height) = (decoded.frame.width, decoded.frame.height); + + if let Some((before, before_width, before_height)) = &previous + && (*before_width, *before_height) == (width, height) + { + let (mut count, mut min_x, mut min_y, mut max_x, mut max_y) = + (0_usize, u32::MAX, u32::MAX, 0_u32, 0_u32); + for (index, (a, b)) in before.iter().zip(&now).enumerate() { + if (i32::from(*a) - i32::from(*b)).abs() > PIXEL_THRESHOLD { + let (x, y) = (index as u32 % width, index as u32 / width); + count += 1; + min_x = min_x.min(x); + max_x = max_x.max(x); + min_y = min_y.min(y); + max_y = max_y.max(y); + } + } + + let share = count as f64 / now.len().max(1) as f64; + if share >= threshold { + match &mut open { + Some(span) if decoded.shown_s - span.end <= MERGE_GAP_S => { + span.end = decoded.shown_s; + span.frames += 1; + span.peak = span.peak.max(share); + span.bounds = ( + span.bounds.0.min(min_x), + span.bounds.1.min(min_y), + span.bounds.2.max(max_x), + span.bounds.3.max(max_y), + ); + } + _ => { + finished.extend(open.take()); + open = Some(Open { + start: decoded.shown_s, + end: decoded.shown_s, + frames: 1, + peak: share, + bounds: (min_x, min_y, max_x, max_y), + }); + } + } + } + } + + previous = Some((now, width, height)); + true + }, + )?; + finished.extend(open.take()); + + let dimensions = previous + .as_ref() + .map(|(_, width, height)| (*width, *height)); + for span in finished { + let (width, height) = dimensions.unwrap_or((1, 1)); + let (w, h) = (f64::from(width), f64::from(height)); + let bbox = [ + f64::from(span.bounds.0) / w, + f64::from(span.bounds.1) / h, + f64::from(span.bounds.2 - span.bounds.0 + 1) / w, + f64::from(span.bounds.3 - span.bounds.1 + 1) / h, + ]; + let events = self + .events() + .iter() + .filter(|event| { + let at = self.video_s(event.recv_mach_ns); + at >= span.start - COINCIDE_S && at <= span.end + COINCIDE_S + }) + .map(|event| event.seq) + .collect(); + + spans.push(ChangeSpan { + display: *display, + start_video_s: span.start, + end_video_s: span.end, + frames: span.frames, + peak_share: span.peak, + bbox, + bbox_points: frame_box.map(|[x, y, frame_width, frame_height]| { + [ + x + bbox[0] * frame_width, + y + bbox[1] * frame_height, + bbox[2] * frame_width, + bbox[3] * frame_height, + ] + }), + events, + }); + } + } + + spans.sort_by(|a, b| { + a.start_video_s + .total_cmp(&b.start_video_s) + .then(a.display.cmp(&b.display)) + }); + + Ok(ChangesData { + from_video_s: from_s, + to_video_s: to_s, + displays: displays.to_vec(), + frames_examined: examined, + spans, + gaps: self.video_gaps(displays, from_s, to_s), + }) + } + + /// Returns the stretches of `from_s..=to_s` (seconds of video time) in which each of + /// `displays` has no video, ordered by display and then by time. + /// + /// Only the manifest is read, no video is decoded. A segment of a display that recorded no + /// frames does not count as video. A stretch shorter than 0.5 s is not a gap, because a display + /// whose screen does not change produces no frames. A display that is not in the session has + /// no video anywhere, so it gets one gap that is the whole range. + /// + /// # Example + /// + /// ```rust + /// use komorebi_recorder::analysis::Analysis; + /// use komorebi_recorder::analysis::changes::Gap; + /// use komorebi_recorder::testkit::SessionBuilder; + /// + /// // a ten second recording that has no video for display 2 + /// let (session, _) = SessionBuilder::new("doc-video-gaps").length(10.0).build(); + /// let analysis = Analysis::open(&session.dir)?; + /// + /// assert_eq!( + /// analysis.video_gaps(&[2], 0.0, 10.0), + /// [Gap { display: 2, from_video_s: 0.0, to_video_s: 10.0 }], + /// ); + /// # Ok::<(), Box>(()) + /// ``` + /// + /// A display that has video for part of the range, and a gap after it, is covered by + /// `tests/video.rs::a_display_with_no_video_for_part_of_the_range_has_a_gap`. + #[must_use] + pub fn video_gaps(&self, displays: &[u32], from_s: f64, to_s: f64) -> Vec { + let mut gaps = Vec::new(); + for display in displays { + let mut covered: Vec<(f64, f64)> = self + .manifest + .displays + .iter() + .filter(|entry| entry.result.display.id == *display && entry.result.frames > 0) + .map(|entry| { + ( + entry.offset_s, + entry.end_s.unwrap_or_else(|| self.manifest.video_span().1), + ) + }) + .collect(); + covered.sort_by(|a, b| a.0.total_cmp(&b.0)); + + let mut cursor = from_s; + for (start, end) in covered { + if start > cursor + MIN_GAP_S && cursor < to_s { + gaps.push(Gap { + display: *display, + from_video_s: cursor, + to_video_s: start.min(to_s), + }); + } + cursor = cursor.max(end); + } + if cursor + MIN_GAP_S < to_s { + gaps.push(Gap { + display: *display, + from_video_s: cursor, + to_video_s: to_s, + }); + } + } + + gaps + } +} diff --git a/komorebi-recorder/src/analysis/mod.rs b/komorebi-recorder/src/analysis/mod.rs index b51360ff..173d34d4 100644 --- a/komorebi-recorder/src/analysis/mod.rs +++ b/komorebi-recorder/src/analysis/mod.rs @@ -12,6 +12,7 @@ pub mod blocks; pub mod brief; pub mod budget; pub mod catalog; +pub mod changes; pub mod classify; pub mod code; pub mod diff; @@ -28,9 +29,11 @@ pub mod lookup; pub mod overview; pub mod schemas; pub mod search; +pub mod sheet; pub mod state; pub mod timespec; pub mod trace; +pub mod video; pub mod views; pub use facade::Analysis; diff --git a/komorebi-recorder/src/analysis/schemas.rs b/komorebi-recorder/src/analysis/schemas.rs index a132b06e..2eaa413b 100644 --- a/komorebi-recorder/src/analysis/schemas.rs +++ b/komorebi-recorder/src/analysis/schemas.rs @@ -16,11 +16,13 @@ use super::answers::DiffData; use super::answers::EventsData; use super::answers::StateData; use super::brief::BriefData; +use super::changes::ChangesData; use super::code::CodeData; use super::envelope::Envelope; use super::history::HistoryData; use super::overview::OverviewData; use super::search::SearchData; +use super::sheet::SheetData; use super::trace::TraceData; use schemars::schema_for; use serde_json::Value; @@ -92,6 +94,16 @@ pub const ALL: &[Entry] = &[ about: "`search`: where a word appears in the recording", schema: of::, }, + Entry { + name: "changes/1", + about: "`changes`: when and where the picture changed on each display", + schema: of::, + }, + Entry { + name: "sheet/1", + about: "`sheet`: a contact sheet image and where each frame is on it", + schema: of::, + }, Entry { name: "annotation/1", about: "`annotate add|edit`: the annotation as it now reads", diff --git a/komorebi-recorder/src/analysis/sheet.rs b/komorebi-recorder/src/analysis/sheet.rs new file mode 100644 index 00000000..871cf87b --- /dev/null +++ b/komorebi-recorder/src/analysis/sheet.rs @@ -0,0 +1,549 @@ +//! A contact sheet: frames of the recording laid out on one image, with the times, the events and +//! the bug regions marked. +//! +//! It is for someone (or an agent that can look at an image) to see what the screen did without +//! playing a video. +//! +//! Each display is its own block of frames in time order. Every frame says when it is; where +//! komorebi events fall around it they are named; frames inside a bug region the user marked have +//! a red border. Frames are decoded at the width they are drawn, so a sheet of a long recording +//! costs little. +//! +//! [`Analysis::sheet`] draws one from [`SheetOptions`] and a `SessionVideo` (the decoder of the +//! session's video, in `analysis::video`) and returns a [`SheetData`] saying where each frame is on +//! the image, so that a reader of the answer can find a frame without looking at the picture. The +//! image is a PNG written to [`SheetOptions::out`]. Decoding needs a host that can decode the video +//! (macOS and Windows; see `crate::av::supported`). +//! +//! # Layout +//! +//! From the top: a title with the session's directory name and the first and last of the moments; +//! under it, if a bug region overlaps those moments, a red line with the first such region's id and +//! its description (cut to 150 characters); then, for each display with a frame to show, a header +//! naming it and its frames in rows of [`SheetOptions::columns`]. Under each frame is its timecode +//! (red, with `BUG`, if it is inside a region) and the first two events received near it, as `# +//! `, with `+` for any more. An event is near a frame if it was received within half the +//! average spacing of the moments (half a second for a single moment) of the frame's moment, but +//! never less than a tenth of a second. +//! +//! The drawn width of a frame is [`SheetOptions::cell_width`], limited to between 64 and 1920 +//! pixels, and the number of columns is limited to the number of moments. The image may be at most +//! 8192 pixels wide and 16384 tall; a sheet that would be larger is refused (see +//! [`Analysis::sheet`]). +//! +//! # Example +//! +//! Needs `ffmpeg` to make a test video, and a host that can decode it, so it is only compiled here; +//! the tests `a_sheet_shows_the_frame_that_was_on_screen_at_each_moment` and +//! `a_sheet_marks_the_bug_region_and_names_the_events_around_a_frame` in `tests/video.rs` run the +//! same call through the `sheet` command. +//! +//! ```rust,no_run +//! use komorebi_recorder::analysis::Analysis; +//! use komorebi_recorder::analysis::sheet::SheetOptions; +//! use komorebi_recorder::analysis::video::SessionVideo; +//! use komorebi_recorder::testkit::SessionBuilder; +//! +//! // a recording of one display, 300 frames at 30 frames per second +//! let (session, _) = SessionBuilder::new("doc-sheet").video(1, 0.0, 300, 30).build(); +//! let analysis = Analysis::open(&session.dir)?; +//! let video = SessionVideo::new(&analysis.manifest, analysis.guard()); +//! +//! let sheet = analysis.sheet(&video, &SheetOptions { +//! times: vec![1.0, 5.0, 9.0], +//! displays: video.displays(), +//! cell_width: 240, +//! columns: 3, +//! regions: Vec::new(), +//! out: session.path("sheets/example.png"), +//! })?; +//! +//! assert_eq!(sheet.frames.len(), 3); +//! assert!(sheet.skipped.is_empty()); +//! # Ok::<(), Box>(()) +//! ``` + +use super::Analysis; +use super::envelope; +use super::video::SessionVideo; +use crate::av::Scale; +use crate::timecode; +use ab_glyph::Font; +use ab_glyph::FontRef; +use ab_glyph::PxScale; +use ab_glyph::ScaleFont; +use ab_glyph::point; +use color_eyre::eyre; +use color_eyre::eyre::WrapErr; +use schemars::JsonSchema; +use serde::Serialize; +use std::io::BufWriter; +use std::path::Path; +use std::path::PathBuf; + +/// The `schema` of the `sheet` command's output: the name that [`SheetData`] has in an envelope. +pub const SCHEMA_SHEET: &str = "sheet/1"; + +const BACKGROUND: [u8; 3] = [24, 24, 28]; +const TEXT: [u8; 3] = [225, 225, 230]; +const DIM: [u8; 3] = [140, 140, 150]; +const EVENT: [u8; 3] = [240, 190, 70]; +const BUG: [u8; 3] = [230, 70, 60]; +const MARGIN: u32 = 10; +const LABEL_HEIGHT: u32 = 44; +const HEADER_HEIGHT: u32 = 26; +const TITLE_HEIGHT: u32 = 44; +const MAX_WIDTH: u32 = 8192; +const MAX_HEIGHT: u32 = 16384; + +/// A bug region to mark on the sheet. +#[derive(Debug, Clone)] +pub struct Region { + /// The annotation's id, written on the sheet and in [`SheetFrame::bugs`]. + pub id: String, + /// Where the region starts, in seconds of video time. + pub start_s: f64, + /// Where the region ends, in seconds of video time. The `sheet` command gives an open-ended + /// bug the end of the recording. + pub end_s: f64, + /// What the user said is wrong, written under the title of the sheet. + pub text: String, +} + +/// What to put on a sheet, and where to write it. +#[derive(Debug, Clone)] +pub struct SheetOptions { + /// The moments to show, in session video time. They are drawn in this order, so a caller that + /// wants them in time order sorts them. Must not be empty. + pub times: Vec, + /// The displays to show, each as a block of its own. Must not be empty; a display listed that + /// has no video at any of the moments is left out. + pub displays: Vec, + /// How wide each frame is drawn, in pixels; limited to between 64 and 1920. + pub cell_width: u32, + /// How many frames to a row; limited to between 1 and the number of moments. + pub columns: usize, + /// The bug regions to mark: a frame whose moment is inside one has a red border. + pub regions: Vec, + /// Where to write the PNG. Its parent directories are created, and a file already there is + /// replaced. + pub out: PathBuf, +} + +/// One frame of a sheet and where it is on the image. +#[derive(Debug, Clone, PartialEq, Serialize, JsonSchema)] +pub struct SheetFrame { + /// The display the frame is of. + pub display: u32, + /// The moment that was asked for, in seconds of video time. + pub requested_video_s: f64, + /// When the frame shown was actually put on screen, in seconds of video time. It is the picture + /// that was on screen at the requested moment, so it is never later than it, and can be + /// earlier: a still screen produces no new frames. + pub shown_video_s: f64, + /// Where it is on the image: pixels from the left edge to its left edge. + pub x: u32, + /// Where it is on the image: pixels from the top edge to its top edge. + pub y: u32, + /// How wide it is drawn, in pixels. + pub width: u32, + /// How tall it is drawn, in pixels. + pub height: u32, + /// The sequence numbers of the events received near it. + pub events: Vec, + /// The ids of the bug regions it is inside. + pub bugs: Vec, +} + +/// A moment of a display that the sheet has no frame for. +#[derive(Debug, Clone, PartialEq, Serialize, JsonSchema)] +pub struct Skipped { + /// The display that was asked for. + pub display: u32, + /// The moment that was asked for, in seconds of video time. + pub video_s: f64, + /// Why there is no frame. + pub reason: String, +} + +/// The answer to `sheet`: the image that was written and where each frame is on it. +#[derive(Debug, Clone, PartialEq, Serialize, JsonSchema)] +pub struct SheetData { + /// Where the PNG was written. + pub path: String, + /// The width of the image, in pixels. + pub width: u32, + /// The height of the image, in pixels. + pub height: u32, + /// The frames on the image, a display at a time and in the order of the moments within it. + pub frames: Vec, + /// Moments a display had no video. + pub skipped: Vec, +} + +struct Canvas { + width: u32, + height: u32, + rgba: Vec, +} + +impl Canvas { + fn new(width: u32, height: u32) -> Self { + let mut rgba = Vec::with_capacity(width as usize * height as usize * 4); + for _ in 0..width as usize * height as usize { + rgba.extend_from_slice(&[BACKGROUND[0], BACKGROUND[1], BACKGROUND[2], 255]); + } + Self { + width, + height, + rgba, + } + } + + fn put(&mut self, x: i64, y: i64, color: [u8; 3], alpha: f32) { + if x < 0 || y < 0 || x >= i64::from(self.width) || y >= i64::from(self.height) { + return; + } + + let at = (y as usize * self.width as usize + x as usize) * 4; + for (channel, wanted) in color.iter().enumerate() { + let below = f32::from(self.rgba[at + channel]); + self.rgba[at + channel] = (below + (f32::from(*wanted) - below) * alpha).round() as u8; + } + } + + fn rectangle(&mut self, x: u32, y: u32, width: u32, height: u32, color: [u8; 3]) { + for row in y..y + height { + for column in x..x + width { + self.put(i64::from(column), i64::from(row), color, 1.0); + } + } + } + + fn border(&mut self, x: u32, y: u32, width: u32, height: u32, thickness: u32, color: [u8; 3]) { + self.rectangle(x, y, width, thickness, color); + self.rectangle(x, y + height - thickness, width, thickness, color); + self.rectangle(x, y, thickness, height, color); + self.rectangle(x + width - thickness, y, thickness, height, color); + } + + /// Copy a BGRA frame in at `x`, `y`. + fn blit(&mut self, x: u32, y: u32, width: u32, height: u32, bgra: &[u8]) { + for row in 0..height { + for column in 0..width { + let from = (row as usize * width as usize + column as usize) * 4; + if from + 2 >= bgra.len() { + return; + } + self.put( + i64::from(x + column), + i64::from(y + row), + [bgra[from + 2], bgra[from + 1], bgra[from]], + 1.0, + ); + } + } + } + + fn text(&mut self, font: &FontRef, size: f32, x: u32, y: u32, text: &str, color: [u8; 3]) { + let scaled = font.as_scaled(PxScale::from(size)); + let baseline = y as f32 + scaled.ascent(); + let mut caret = x as f32; + for character in text.chars() { + let id = scaled.glyph_id(character); + let glyph = id.with_scale_and_position(PxScale::from(size), point(caret, baseline)); + caret += scaled.h_advance(id); + if let Some(outlined) = font.outline_glyph(glyph) { + let bounds = outlined.px_bounds(); + outlined.draw(|gx, gy, coverage| { + self.put( + i64::from(bounds.min.x as i32) + i64::from(gx), + i64::from(bounds.min.y as i32) + i64::from(gy), + color, + coverage, + ); + }); + } + } + } + + fn save(&self, path: &Path) -> eyre::Result<()> { + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent) + .wrap_err_with(|| format!("could not create {}", parent.display()))?; + } + + let file = std::fs::File::create(path) + .wrap_err_with(|| format!("could not create {}", path.display()))?; + let mut encoder = png::Encoder::new(BufWriter::new(file), self.width, self.height); + encoder.set_color(png::ColorType::Rgba); + encoder.set_depth(png::BitDepth::Eight); + let mut writer = encoder.write_header()?; + writer.write_image_data(&self.rgba)?; + writer.finish()?; + Ok(()) + } +} + +fn truncate(text: &str, limit: usize) -> String { + let single: String = text.split_whitespace().collect::>().join(" "); + if single.chars().count() <= limit { + single + } else { + let mut cut: String = single.chars().take(limit.saturating_sub(1)).collect(); + cut.push('…'); + cut + } +} + +impl Analysis { + /// Draws a contact sheet of the moments in `options` and writes it as a PNG. + /// + /// Each frame is the one that was on screen at its moment, decoded at the drawn width. A moment + /// at which a display had no video (it was unplugged, asleep, or not yet recorded) is skipped + /// and listed in [`SheetData::skipped`] rather than failing the sheet. See the module docs for + /// the layout. + /// + /// # Errors + /// + /// - a usage error if `options.times` is empty, or if the sheet would be wider than 8192 or + /// taller than 16384 pixels (the message says how big it would be; show fewer frames or draw + /// them narrower); + /// - an unusable error if `options.displays` is empty, which is what the commands pass for a + /// session that recorded no video; + /// - a not-found error if none of the moments has video on any of the displays; + /// - an error if a video file cannot be opened or decoded (including on a host that cannot + /// decode video), if the output directory or file cannot be created, or if the PNG cannot be + /// written. + /// + /// # Example + /// + /// See the [module docs](self). + pub fn sheet(&self, video: &SessionVideo, options: &SheetOptions) -> eyre::Result { + if options.times.is_empty() { + return Err(envelope::usage("there are no moments to show")); + } + if options.displays.is_empty() { + return Err(envelope::unusable("the session recorded no video")); + } + + let font = FontRef::try_from_slice(epaint_default_fonts::HACK_REGULAR) + .map_err(|_| eyre::Report::msg("the built-in font could not be read"))?; + let columns = options.columns.clamp(1, options.times.len()); + let cell = options.cell_width.clamp(64, 1920); + + // decode every frame first, so the layout knows each one's height + struct Cell { + requested: f64, + shown: f64, + width: u32, + height: u32, + bgra: Vec, + } + let mut blocks: Vec<(u32, Vec)> = Vec::new(); + let mut skipped = Vec::new(); + for display in &options.displays { + let mut cells = Vec::new(); + for time in &options.times { + match video.frame_at(*display, *time, Scale::Width(cell))? { + Some(decoded) => cells.push(Cell { + requested: *time, + shown: decoded.shown_s, + width: decoded.frame.width, + height: decoded.frame.height, + bgra: decoded.frame.bgra, + }), + None => skipped.push(Skipped { + display: *display, + video_s: *time, + reason: + "the display had no video then (unplugged, asleep, or not yet recorded)" + .to_string(), + }), + } + } + if !cells.is_empty() { + blocks.push((*display, cells)); + } + } + if blocks.is_empty() { + return Err(envelope::not_found( + "none of those moments has video on any display", + )); + } + + let width = MARGIN + columns as u32 * (cell + MARGIN); + let mut height = TITLE_HEIGHT + MARGIN; + for (_, cells) in &blocks { + height += HEADER_HEIGHT; + for row in cells.chunks(columns) { + height += + row.iter().map(|cell| cell.height).max().unwrap_or(0) + LABEL_HEIGHT + MARGIN; + } + } + if width > MAX_WIDTH || height > MAX_HEIGHT { + return Err(envelope::usage(format!( + "a sheet of {} frames at width {cell} would be {width}x{height}: show fewer frames \ + or draw them narrower", + options.times.len() + ))); + } + + let mut canvas = Canvas::new(width, height); + let first = options.times.iter().copied().fold(f64::INFINITY, f64::min); + let last = options + .times + .iter() + .copied() + .fold(f64::NEG_INFINITY, f64::max); + let session_name = self + .session + .dir + .file_name() + .map_or_else(String::new, |name| name.to_string_lossy().into_owned()); + canvas.text( + &font, + 15.0, + MARGIN, + 6, + &format!( + "{session_name} {} - {}", + timecode::format(first), + timecode::format(last) + ), + TEXT, + ); + if let Some(region) = options + .regions + .iter() + .find(|region| region.end_s >= first && region.start_s <= last) + { + canvas.text( + &font, + 12.0, + MARGIN, + 26, + &format!("BUG {}: {}", region.id, truncate(®ion.text, 150)), + BUG, + ); + } + + // spread of the moments, to know how near an event has to be to belong to a frame + let step = if options.times.len() > 1 { + (last - first) / (options.times.len() - 1) as f64 + } else { + 1.0 + }; + let reach = (step / 2.0).max(0.1); + + let mut frames = Vec::new(); + let mut y = TITLE_HEIGHT + MARGIN; + for (display, cells) in &blocks { + canvas.text( + &font, + 14.0, + MARGIN, + y + 4, + &format!("display {display}"), + DIM, + ); + y += HEADER_HEIGHT; + + for row in cells.chunks(columns) { + let row_height = row.iter().map(|cell| cell.height).max().unwrap_or(0); + for (column, cell_frame) in row.iter().enumerate() { + let x = MARGIN + column as u32 * (cell + MARGIN); + canvas.blit(x, y, cell_frame.width, cell_frame.height, &cell_frame.bgra); + + let bugs: Vec = options + .regions + .iter() + .filter(|region| { + cell_frame.requested >= region.start_s - 1e-6 + && cell_frame.requested <= region.end_s + 1e-6 + }) + .map(|region| region.id.clone()) + .collect(); + if !bugs.is_empty() { + canvas.border(x, y, cell_frame.width, cell_frame.height, 3, BUG); + } + + let events: Vec = self + .events() + .iter() + .filter(|event| { + (self.video_s(event.recv_mach_ns) - cell_frame.requested).abs() <= reach + }) + .map(|event| event.seq) + .collect(); + + let label_y = y + row_height + 3; + canvas.text( + &font, + 13.0, + x, + label_y, + &timecode::format(cell_frame.requested), + if bugs.is_empty() { TEXT } else { BUG }, + ); + if !bugs.is_empty() { + canvas.text(&font, 12.0, x + 92, label_y, "BUG", BUG); + } + let named: Vec = events + .iter() + .take(2) + .filter_map(|seq| { + let index = self.event_index(*seq)?; + let event = &self.events()[index]; + Some(format!("#{seq} {}", event.info.name)) + }) + .collect(); + if !named.is_empty() { + let more = events.len().saturating_sub(2); + canvas.text( + &font, + 11.0, + x, + label_y + 16, + &truncate( + &format!( + "{}{}", + named.join(" "), + if more > 0 { + format!(" +{more}") + } else { + String::new() + } + ), + (cell / 7) as usize, + ), + EVENT, + ); + } + + frames.push(SheetFrame { + display: *display, + requested_video_s: cell_frame.requested, + shown_video_s: cell_frame.shown, + x, + y, + width: cell_frame.width, + height: cell_frame.height, + events, + bugs, + }); + } + y += row_height + LABEL_HEIGHT + MARGIN; + } + } + + canvas.save(&options.out)?; + Ok(SheetData { + path: options.out.display().to_string(), + width, + height, + frames, + skipped, + }) + } +} diff --git a/komorebi-recorder/src/analysis/video.rs b/komorebi-recorder/src/analysis/video.rs new file mode 100644 index 00000000..ced19039 --- /dev/null +++ b/komorebi-recorder/src/analysis/video.rs @@ -0,0 +1,354 @@ +//! The recorded video, addressed by session time and display. +//! +//! A display's video is one file per segment (a display that was unplugged or slept and came +//! back starts a new one), and a file's own time starts at its first frame. This maps a moment of +//! session video time on a display to the file and file time that has it, decodes frames, and says +//! when there is no video to show (the display was absent). +//! +//! [`SessionVideo`] is what the `changes` and `sheet` commands and the viewer's playback ask for +//! frames. It uses the decoder of `crate::av` (`AVFoundation` on macOS, Media Foundation on Windows), +//! so a host with neither cannot open a file; see `crate::av::supported`. +//! +//! # Times +//! +//! Every time here is *session video time*: seconds from the first frame of any display +//! ([`crate::session::Manifest::video_t0`]). A segment's file has its own time, which starts at its +//! first frame, and the manifest says how far into session time that is ([`DisplayEntry::offset_s`]): +//! session time `t` is at `t - offset_s` in the file. A moment is covered by a segment of a display +//! if it falls between the segment's start and end, with 50 ms of slack on each side, since +//! segment boundaries are sampled and not exact; where two segments of a display both cover a +//! moment, the later one is used ([`crate::session::Manifest::segment_at`]). A moment covered by +//! none is a gap (the display was unplugged, asleep, or not yet recorded) and the methods say so +//! with `None`, never with an invented frame. +//! +//! | To get | Use | Crosses segments | +//! |---|---|---| +//! | the frame on screen at one moment | [`SessionVideo::frame_at`] | no | +//! | the frames from one moment on, one at a time | [`SessionVideo::open_at`] | no | +//! | every frame of a range | [`SessionVideo::scan`] | yes | +//! | which displays have video, and whether one does at a moment | [`SessionVideo::displays`], [`SessionVideo::has_video_at`] | | +//! +//! # Example +//! +//! Needs `ffmpeg` to make a test video, and a host that can decode it, so it is only compiled here; +//! the tests of `tests/video.rs` (`a_sheet_shows_the_frame_that_was_on_screen_at_each_moment`, +//! `a_moment_a_display_had_no_video_is_skipped_not_invented`) run these calls through the `sheet` +//! command. +//! +//! ```rust,no_run +//! use komorebi_recorder::analysis::Analysis; +//! use komorebi_recorder::analysis::video::SessionVideo; +//! use komorebi_recorder::av::Scale; +//! use komorebi_recorder::testkit::SessionBuilder; +//! +//! // display 1 has video for 3 seconds from the start; display 2 only from 1 s to 3 s +//! let (session, _) = SessionBuilder::new("doc-video") +//! .video(1, 0.0, 90, 30) +//! .video(2, 1.0, 60, 30) +//! .build(); +//! let analysis = Analysis::open(&session.dir)?; +//! let video = SessionVideo::new(&analysis.manifest, analysis.guard()); +//! +//! assert_eq!(video.displays(), [1, 2]); +//! assert!(video.has_video_at(1, 0.5)); +//! assert!(!video.has_video_at(2, 0.5)); +//! +//! // the frame on screen at 1.5 s, at most 128 pixels wide; never shown later than asked +//! let decoded = video.frame_at(1, 1.5, Scale::Width(128))?.expect("display 1 had video"); +//! assert!(decoded.shown_s <= 1.5); +//! +//! // display 2 had none at 0.5 s +//! assert!(video.frame_at(2, 0.5, Scale::Width(128))?.is_none()); +//! # Ok::<(), Box>(()) +//! ``` + +use super::guard::SessionPath; +use crate::av::Asset; +use crate::av::Frame; +use crate::av::Scale; +use crate::session::DisplayEntry; +use crate::session::Manifest; +use color_eyre::eyre; +use std::cell::RefCell; +use std::collections::HashMap; +use std::path::PathBuf; +use std::rc::Rc; + +/// A decoded frame and when it was actually on screen. +pub struct DecodedFrame { + /// The display the frame is of. + pub display: u32, + /// The segment of the display's video it came from. + pub segment: u32, + /// When the frame was shown, in session video time. Never later than what was asked for. + /// + /// A screen that holds still produces no new frames, so the frame that was on screen at a + /// moment can have been put there some time before it. + pub shown_s: f64, + /// The decoded picture, scaled as asked. + pub frame: Frame, +} + +/// A started decode of one segment of one display. Frames come in order, each with the session +/// video time it was on screen. +/// +/// Made by [`SessionVideo::open_at`]. A stream stays within its segment: it does not continue into +/// the next one. +pub struct Stream { + reader: crate::av::Reader, + offset_s: f64, + /// The segment of the display's video it reads. + pub segment: u32, + /// Session video time at which the segment ends. + pub end_s: f64, +} + +impl Stream { + /// Returns the next frame and when it was shown, in session video time, or `None` at the end of + /// the segment (or of the range the stream was opened with). + /// + /// # Errors + /// + /// Returns an error if the decoder fails. + pub fn next_frame(&mut self) -> eyre::Result> { + Ok(self + .reader + .next_frame()? + .map(|frame| (self.offset_s + frame.pts_s, frame))) + } +} + +/// The video of a session. Not `Send`: decoding objects belong to the thread that made them. +/// +/// A file is opened the first time a frame of it is asked for and kept open for the life of the +/// `SessionVideo`, so asking for many frames of the same segment opens it once. It borrows the +/// session's [`Manifest`] and [`SessionPath`] (`Analysis::manifest` and `Analysis::guard`), and the +/// guard is what keeps a file named in the manifest from leading outside the session. +/// +/// See the [module docs](self) for how session time maps to files and an example. +/// +/// # Example: it stays on the thread that made it +/// +/// The compiler refuses to send one to another thread: +/// +/// ```rust,compile_fail,E0277 +/// use komorebi_recorder::analysis::video::SessionVideo; +/// +/// fn send_to_another_thread() {} +/// send_to_another_thread::>(); +/// ``` +/// +/// A thread that decodes makes its own, from a manifest and a guard that can be shared (an +/// [`Analysis`](super::Analysis) is `Send` and `Sync`): +/// +/// ```rust +/// use komorebi_recorder::analysis::Analysis; +/// use komorebi_recorder::analysis::video::SessionVideo; +/// use komorebi_recorder::testkit::SessionBuilder; +/// use std::sync::Arc; +/// +/// let (session, _) = SessionBuilder::new("doc-video-threads").build(); +/// let analysis = Arc::new(Analysis::open(&session.dir)?); +/// +/// let shared = Arc::clone(&analysis); +/// let displays = std::thread::spawn(move || { +/// SessionVideo::new(&shared.manifest, shared.guard()).displays() +/// }) +/// .join() +/// .expect("the thread finished"); +/// assert!(displays.is_empty(), "a session recorded without video has no displays with video"); +/// # Ok::<(), Box>(()) +/// ``` +pub struct SessionVideo<'a> { + manifest: &'a Manifest, + guard: &'a SessionPath, + assets: RefCell>>, +} + +impl<'a> SessionVideo<'a> { + /// Returns the video of the session described by `manifest`, resolving its files through + /// `guard`. Nothing is opened until a frame is asked for. + #[must_use] + pub fn new(manifest: &'a Manifest, guard: &'a SessionPath) -> Self { + Self { + manifest, + guard, + assets: RefCell::new(HashMap::new()), + } + } + + /// Returns the ids of the displays that have any video (a segment with at least one frame), in + /// ascending order. + #[must_use] + pub fn displays(&self) -> Vec { + let mut ids: Vec = self + .manifest + .displays + .iter() + .filter(|entry| entry.result.frames > 0) + .map(|entry| entry.result.display.id) + .collect(); + ids.sort_unstable(); + ids.dedup(); + ids + } + + fn asset(&self, entry: &DisplayEntry) -> eyre::Result> { + let path = self.guard.resolve(&entry.result.file)?; + if let Some(found) = self.assets.borrow().get(&path) { + return Ok(found.clone()); + } + + let asset = Rc::new(Asset::open(&path)?); + self.assets.borrow_mut().insert(path, asset.clone()); + Ok(asset) + } + + /// Returns the frame of `display` on screen at session video time `video_s`, decoded at + /// `scale`, or `None` if the display had no video then. + /// + /// The frame's [`DecodedFrame::shown_s`] is the time it was put on screen, which is never later + /// than `video_s` and can be earlier. A time at or past the end of the file asks for its last + /// frame, not for something beyond it. `None` is also returned if the decoder produces no frame + /// for the moment. The display's video is looked up as [`SessionVideo`]'s module docs describe, + /// so a moment in the slack at the edge of a segment is answered with that segment's nearest + /// frame. + /// + /// # Errors + /// + /// Returns an error if the segment's file leads outside the session (a usage error), cannot be + /// opened (it is missing or damaged, or the host cannot decode video at all), or cannot be + /// decoded. + pub fn frame_at( + &self, + display: u32, + video_s: f64, + scale: Scale, + ) -> eyre::Result> { + let Some(entry) = self.manifest.segment_at(display, video_s) else { + return Ok(None); + }; + + let asset = self.asset(entry)?; + // the last moment of a file is the last frame, not past it + let file_s = (video_s - entry.offset_s) + .max(0.0) + .min((asset.duration_s() - 0.002).max(0.0)); + let mut reader = asset.start(file_s, Some(file_s + 0.5), scale)?; + Ok(reader.next_frame()?.map(|frame| DecodedFrame { + display, + segment: entry.result.segment, + shown_s: entry.offset_s + frame.pts_s, + frame, + })) + } + + /// Returns `true` if and only if `display` has video at session video time `video_s`: a segment + /// of it covers the moment (see the [module docs](self)). It does not open any file. + #[must_use] + pub fn has_video_at(&self, display: u32, video_s: f64) -> bool { + self.manifest.segment_at(display, video_s).is_some() + } + + /// Starts decoding the segment of `display` that has video at `video_s`, beginning with the + /// frame on screen then and running to `to_s` (or the end of the segment). `None` if the + /// display had no video then. + /// + /// The stream ends at the end of the segment even if `to_s` is later: use + /// [`SessionVideo::scan`] to read across segments. A `to_s` before `video_s` is read as + /// `video_s`. + /// + /// # Errors + /// + /// Returns an error under the same conditions as [`SessionVideo::frame_at`]. + pub fn open_at( + &self, + display: u32, + video_s: f64, + to_s: Option, + scale: Scale, + ) -> eyre::Result> { + let Some(entry) = self.manifest.segment_at(display, video_s) else { + return Ok(None); + }; + + let asset = self.asset(entry)?; + let start = (video_s - entry.offset_s) + .max(0.0) + .min((asset.duration_s() - 0.002).max(0.0)); + let stop = to_s.map(|to| (to - entry.offset_s).max(start)); + Ok(Some(Stream { + reader: asset.start(start, stop, scale)?, + offset_s: entry.offset_s, + segment: entry.result.segment, + end_s: entry.end_s.unwrap_or_else(|| self.manifest.video_span().1), + })) + } + + /// Decodes every frame of `display` from `from_s` to `to_s`, across segments, in order, calling + /// `visit` with each until it returns `false`. Returns how many frames were visited, counting + /// the one `visit` stopped on. + /// + /// A frame shown before `from_s` is visited only if it was shown less than half a second before + /// it: the decoder starts at the frame that was on screen at `from_s`, which can be earlier than + /// it, and that frame is not visited if it is older than that. No frame shown after `to_s` is + /// visited. A gap between segments is not visited: there is nothing to decode there. + /// + /// # Errors + /// + /// Returns an error under the same conditions as [`SessionVideo::frame_at`], for the first + /// segment that fails. Frames visited before it stay visited. + pub fn scan( + &self, + display: u32, + from_s: f64, + to_s: f64, + scale: Scale, + visit: &mut dyn FnMut(&DecodedFrame) -> bool, + ) -> eyre::Result { + let mut segments: Vec<&DisplayEntry> = self + .manifest + .displays + .iter() + .filter(|entry| entry.result.display.id == display && entry.result.frames > 0) + .collect(); + segments.sort_by(|a, b| a.offset_s.total_cmp(&b.offset_s)); + let (_, session_end) = self.manifest.video_span(); + + let mut visited = 0; + for entry in segments { + let end = entry.end_s.unwrap_or(session_end); + if end < from_s || entry.offset_s > to_s { + continue; + } + + let asset = self.asset(entry)?; + let start = (from_s - entry.offset_s).max(0.0); + let stop = (to_s - entry.offset_s).min(asset.duration_s()); + let mut reader = asset.start(start, Some(stop), scale)?; + while let Some(frame) = reader.next_frame()? { + let shown_s = entry.offset_s + frame.pts_s; + // the decoder starts at the frame on screen at `from_s`, which can be before it + if shown_s + 1e-6 < from_s && shown_s + 0.5 < from_s { + continue; + } + if shown_s > to_s { + break; + } + + visited += 1; + let decoded = DecodedFrame { + display, + segment: entry.result.segment, + shown_s, + frame, + }; + if !visit(&decoded) { + return Ok(visited); + } + } + } + + Ok(visited) + } +} diff --git a/komorebi-recorder/src/cli/mod.rs b/komorebi-recorder/src/cli/mod.rs index c6a1ce9a..a1ae05b1 100644 --- a/komorebi-recorder/src/cli/mod.rs +++ b/komorebi-recorder/src/cli/mod.rs @@ -11,6 +11,7 @@ mod doctor; mod investigate; mod query; mod record; +mod video; use crate::analysis::envelope::unusable; use crate::analysis::envelope::with_hint; @@ -43,6 +44,8 @@ pub use query::LocateArgs; pub use query::RangeArgs; pub use query::SummaryArgs; pub use record::RecordArgs; +pub use video::ChangesArgs; +pub use video::SheetArgs; /// The command line of `komorebi-recorder`. #[derive(Parser)] @@ -105,6 +108,10 @@ pub enum Command { Window(WindowArgs), /// One event end to end: what its handler logged, what changed, and who changed it Trace(TraceArgs), + /// When and where on each display the picture changed + Changes(ChangesArgs), + /// A contact sheet image of frames, with the times, events and bug region marked + Sheet(SheetArgs), /// Which line of komorebi's source wrote a line of its log, with the code around it Code(CodeArgs), /// Add, edit or delete a session's bug regions, notes and an agent's findings @@ -171,6 +178,8 @@ impl Command { Self::Code(args) => args.json, Self::Trace(args) => args.json, Self::Window(args) => args.json, + Self::Changes(args) => args.json, + Self::Sheet(args) => args.json, Self::Annotate(args) => args.json(), Self::Annotations(args) => args.json, Self::Doctor(args) => args.json, @@ -229,6 +238,8 @@ pub fn run(command: Command) -> eyre::Result<()> { Command::Code(args) => investigate::code(&args), Command::Trace(args) => investigate::trace(&args), Command::Window(args) => investigate::window(&args), + Command::Changes(args) => video::changes(&args), + Command::Sheet(args) => video::sheet(&args), Command::Annotate(args) => annotate::annotate(&args), Command::Annotations(args) => annotate::list(&args), Command::Reindex(args) => reindex(&args), diff --git a/komorebi-recorder/src/cli/video.rs b/komorebi-recorder/src/cli/video.rs new file mode 100644 index 00000000..6d89ee35 --- /dev/null +++ b/komorebi-recorder/src/cli/video.rs @@ -0,0 +1,235 @@ +//! `changes` and `sheet`: what the screen did. +//! +//! | Command | Asks | Answer, with `--json` | +//! |---|---|---| +//! | `changes` | [`Analysis::changes`](crate::analysis::Analysis::changes) | `changes/1` | +//! | `sheet` | [`Analysis::sheet`] | `sheet/1` | +//! +//! Both work on a range of video time, given as `--from` and `--to` (any time `state --at` takes; +//! the default is the whole video) or, instead of those, as `--bug `: a bug region from three +//! seconds before it starts to its end. Both work on the displays that have video (`--display`, repeatable, to pick), +//! and both refuse a session that recorded no video. `sheet` also takes `--at` for exact moments, or +//! `--count` moments evenly spaced across the range, and writes a PNG (by default to +//! `/sheets/.png`); see [`crate::analysis::sheet`] for the layout. +//! +//! Needs a host that can decode the video (macOS and Windows; see [`crate::av::supported`]). +//! Covered by `tests/video.rs`, which runs the commands through the real binary on a session with a +//! test video (it skips with a message when `ffmpeg` is not there to make one). + +use super::print_json; +use crate::analysis::Analysis; +use crate::analysis::changes::SCHEMA_CHANGES; +use crate::analysis::envelope; +use crate::analysis::envelope::Envelope; +use crate::analysis::sheet::Region; +use crate::analysis::sheet::SCHEMA_SHEET; +use crate::analysis::sheet::SheetOptions; +use crate::analysis::video::SessionVideo; +use crate::annotations; +use crate::report; +use clap::Args; +use color_eyre::eyre; +use std::path::PathBuf; + +/// Seconds shown before a bug region when a sheet is made of one. +const BUG_PRE_ROLL_S: f64 = 3.0; + +/// The arguments of `changes`. +#[derive(Args, Debug)] +pub struct ChangesArgs { + /// Session directory + pub session: PathBuf, + /// From this moment [default: the start of the video] + #[arg(long, value_name = "TIME")] + pub from: Option, + /// To this moment [default: the end of the video] + #[arg(long, value_name = "TIME")] + pub to: Option, + /// Instead: a bug region, from a few seconds before it to its end + #[arg(long, value_name = "ID", conflicts_with_all = ["from", "to"])] + pub bug: Option, + /// Only this display (repeatable) [default: every display with video] + #[arg(long = "display", value_name = "ID")] + pub displays: Vec, + /// Above 1 notices smaller changes, below 1 only larger ones + #[arg(long, default_value_t = 1.0)] + pub sensitivity: f64, + /// Machine-readable JSON output + #[arg(long)] + pub json: bool, +} + +/// The arguments of `sheet`. +#[derive(Args, Debug)] +pub struct SheetArgs { + /// Session directory + pub session: PathBuf, + /// From this moment [default: the start of the video] + #[arg(long, value_name = "TIME")] + pub from: Option, + /// To this moment [default: the end of the video] + #[arg(long, value_name = "TIME")] + pub to: Option, + /// Instead: a bug region, from a few seconds before it to its end + #[arg(long, value_name = "ID", conflicts_with_all = ["from", "to"])] + pub bug: Option, + /// Show exactly these moments instead of evenly spaced ones (repeatable) + #[arg(long = "at", value_name = "TIME")] + pub at: Vec, + /// How many frames, evenly spaced + #[arg(long, default_value_t = 12)] + pub count: usize, + /// Only this display (repeatable) [default: every display with video] + #[arg(long = "display", value_name = "ID")] + pub displays: Vec, + /// How wide each frame is drawn, in pixels + #[arg(long, default_value_t = 480)] + pub width: u32, + /// Frames per row + #[arg(long, default_value_t = 4)] + pub columns: usize, + /// Where to write the PNG [default: in the session's sheets/ directory, named by the range] + #[arg(long, value_name = "FILE")] + pub out: Option, + /// Machine-readable JSON output + #[arg(long)] + pub json: bool, +} + +/// The video-time range to work on: a bug region, or `--from`/`--to`, or everything. +fn range( + analysis: &Analysis, + from: Option<&str>, + to: Option<&str>, + bug: Option<&str>, +) -> eyre::Result<(f64, f64)> { + if let Some(id) = bug { + let loaded = annotations::load(&analysis.session.dir)?; + let found = loaded + .get(id) + .ok_or_else(|| envelope::not_found(format!("there is no annotation {id}")))?; + let end = found.end_or_session_end(&analysis.manifest); + return Ok(( + (analysis.video_s(found.start_mach_ns) - BUG_PRE_ROLL_S).max(0.0), + analysis.video_s(end), + )); + } + + let resolve = |text: Option<&str>, default: f64| -> eyre::Result { + text.map_or(Ok(default), |text| { + analysis.resolve(text).map(|resolved| resolved.video_s) + }) + }; + let (_, end) = analysis.manifest.video_span(); + let (from, to) = (resolve(from, 0.0)?, resolve(to, end)?); + if to < from { + return Err(envelope::usage("the end of the range is before its start")); + } + + Ok((from, to)) +} + +fn displays(video: &SessionVideo, wanted: &[u32]) -> eyre::Result> { + let available = video.displays(); + if available.is_empty() { + return Err(envelope::unusable("the session recorded no video")); + } + if wanted.is_empty() { + return Ok(available); + } + + for display in wanted { + if !available.contains(display) { + return Err(envelope::not_found(format!( + "display {display} has no video; displays with video: {available:?}" + ))); + } + } + Ok(wanted.to_vec()) +} + +pub(super) fn changes(args: &ChangesArgs) -> eyre::Result<()> { + let analysis = Analysis::open(&args.session)?; + let video = SessionVideo::new(&analysis.manifest, analysis.guard()); + let (from, to) = range( + &analysis, + args.from.as_deref(), + args.to.as_deref(), + args.bug.as_deref(), + )?; + let shown = displays(&video, &args.displays)?; + + let data = analysis.changes(&video, &shown, from, to, args.sensitivity)?; + if args.json { + return print_json(&Envelope::ok(SCHEMA_CHANGES, data).warnings(analysis.warnings())); + } + + print!("{}", report::changes_text(&data)); + Ok(()) +} + +pub(super) fn sheet(args: &SheetArgs) -> eyre::Result<()> { + let analysis = Analysis::open(&args.session)?; + let video = SessionVideo::new(&analysis.manifest, analysis.guard()); + let (from, to) = range( + &analysis, + args.from.as_deref(), + args.to.as_deref(), + args.bug.as_deref(), + )?; + let shown = displays(&video, &args.displays)?; + + let times: Vec = if args.at.is_empty() { + let count = args.count.max(1); + if count == 1 { + vec![from] + } else { + (0..count) + .map(|index| from + (to - from) * index as f64 / (count - 1) as f64) + .collect() + } + } else { + args.at + .iter() + .map(|text| analysis.resolve(text).map(|resolved| resolved.video_s)) + .collect::>()? + }; + + let regions: Vec = annotations::load(&analysis.session.dir)? + .bugs() + .map(|bug| Region { + id: bug.id.clone(), + start_s: analysis.video_s(bug.start_mach_ns), + end_s: analysis.video_s(bug.end_or_session_end(&analysis.manifest)), + text: bug.text.clone(), + }) + .collect(); + + let out = args.out.clone().unwrap_or_else(|| { + analysis.session.path(format!( + "sheets/{}-{}.png", + crate::timecode::format(times.iter().copied().fold(f64::INFINITY, f64::min)) + .replace(':', "-"), + crate::timecode::format(times.iter().copied().fold(f64::NEG_INFINITY, f64::max)) + .replace(':', "-"), + )) + }); + + let data = analysis.sheet( + &video, + &SheetOptions { + times, + displays: shown, + cell_width: args.width, + columns: args.columns, + regions, + out, + }, + )?; + if args.json { + return print_json(&Envelope::ok(SCHEMA_SHEET, data).warnings(analysis.warnings())); + } + + print!("{}", report::sheet_text(&data)); + Ok(()) +} diff --git a/komorebi-recorder/src/report.rs b/komorebi-recorder/src/report.rs index 8702ea0a..88403fbe 100644 --- a/komorebi-recorder/src/report.rs +++ b/komorebi-recorder/src/report.rs @@ -31,6 +31,7 @@ use crate::analysis::answers::EventsData; use crate::analysis::answers::StateData; use crate::analysis::brief::BriefData; use crate::analysis::brief::RegionBrief; +use crate::analysis::changes::ChangesData; use crate::analysis::code::CodeData; use crate::analysis::history::HistoryData; use crate::analysis::invariants::Severity; @@ -38,6 +39,7 @@ use crate::analysis::invariants::Violation; use crate::analysis::overview::Finding; use crate::analysis::overview::OverviewData; use crate::analysis::search::SearchData; +use crate::analysis::sheet::SheetData; use crate::analysis::state::Container; use crate::analysis::state::Window; use crate::analysis::trace::TraceData; @@ -1554,6 +1556,147 @@ pub fn search_text(data: &SearchData) -> String { out } +/// Renders the answer to `changes`: how many frames were compared, then a line for each stretch of +/// change, and the stretches in which a display had no video. +/// +/// A stretch of change has its display, times and length, the most of the screen that changed +/// between two frames, the box around what changed as percentages of the screen, and the events +/// received during it. +/// +/// # Example +/// +/// ```rust +/// use komorebi_recorder::analysis::changes::{ChangeSpan, ChangesData, Gap}; +/// use komorebi_recorder::report::changes_text; +/// +/// let data = ChangesData { +/// from_video_s: 0.0, +/// to_video_s: 4.0, +/// displays: vec![1], +/// frames_examined: 120, +/// spans: vec![ChangeSpan { +/// display: 1, +/// start_video_s: 1.0, +/// end_video_s: 1.5, +/// frames: 15, +/// peak_share: 0.25, +/// bbox: [0.5, 0.0, 0.5, 1.0], +/// bbox_points: None, +/// events: vec![3], +/// }], +/// gaps: vec![Gap { display: 1, from_video_s: 3.0, to_video_s: 4.0 }], +/// }; +/// +/// let text = changes_text(&data); +/// assert!(text.starts_with("120 frames compared on display(s) 1 from 00:00.000 to 00:04.000: 1 stretch(es) of change\n")); +/// assert!(text.contains("up to 25.0% of the screen")); +/// assert!(text.ends_with("display 1 has no video from 00:03.000 to 00:04.000\n")); +/// ``` +#[must_use] +pub fn changes_text(data: &ChangesData) -> String { + let mut out = String::new(); + let _ = writeln!( + out, + "{} frames compared on display(s) {} from {} to {}: {} stretch(es) of change", + data.frames_examined, + data.displays + .iter() + .map(ToString::to_string) + .collect::>() + .join(", "), + timecode::format(data.from_video_s), + timecode::format(data.to_video_s), + data.spans.len() + ); + for span in &data.spans { + let events = if span.events.is_empty() { + String::new() + } else { + format!( + " events {}", + span.events + .iter() + .map(|seq| format!("#{seq}")) + .collect::>() + .join(" ") + ) + }; + let _ = writeln!( + out, + "display {} {} - {} ({:.2} s, {} frames) up to {:.1}% of the screen box {:.0}%,{:.0}% {:.0}%x{:.0}%{events}", + span.display, + timecode::format(span.start_video_s), + timecode::format(span.end_video_s), + span.end_video_s - span.start_video_s, + span.frames, + span.peak_share * 100.0, + span.bbox[0] * 100.0, + span.bbox[1] * 100.0, + span.bbox[2] * 100.0, + span.bbox[3] * 100.0, + ); + } + for gap in &data.gaps { + let _ = writeln!( + out, + "display {} has no video from {} to {}", + gap.display, + timecode::format(gap.from_video_s), + timecode::format(gap.to_video_s) + ); + } + out +} + +/// Renders the answer to `sheet`: where the image was written, its size and how many frames it +/// has, then a line for each moment that had no frame and why. +/// +/// # Example +/// +/// ```rust +/// use komorebi_recorder::analysis::sheet::{SheetData, Skipped}; +/// use komorebi_recorder::report::sheet_text; +/// +/// let data = SheetData { +/// path: "sheets/00-01.000-00-09.000.png".to_string(), +/// width: 1450, +/// height: 560, +/// frames: Vec::new(), +/// skipped: vec![Skipped { +/// display: 2, +/// video_s: 0.5, +/// reason: "the display had no video then".to_string(), +/// }], +/// }; +/// +/// assert_eq!( +/// sheet_text(&data), +/// "sheets/00-01.000-00-09.000.png (1450x560, 0 frames)\nno frame: display 2 at 00:00.500: the display had no video then\n", +/// ); +/// ``` +#[must_use] +pub fn sheet_text(data: &SheetData) -> String { + let mut out = String::new(); + let _ = writeln!( + out, + "{} ({}x{}, {} frames)", + data.path, + data.width, + data.height, + data.frames.len() + ); + for skipped in &data.skipped { + let _ = writeln!( + out, + "no frame: display {} at {}: {}", + skipped.display, + timecode::format(skipped.video_s), + skipped.reason + ); + } + out +} + #[cfg(test)] mod tests { use super::*; diff --git a/komorebi-recorder/tests/video.rs b/komorebi-recorder/tests/video.rs new file mode 100644 index 00000000..00f31cb9 --- /dev/null +++ b/komorebi-recorder/tests/video.rs @@ -0,0 +1,286 @@ +//! `changes` and `sheet` through the real binary, on sessions whose video says which frame it is. + +use komorebi_recorder::testkit; +use komorebi_recorder::testkit::SessionBuilder; +use serde_json::Value; +use serde_json::json; +use std::process::Command; +use std::process::Output; + +fn recorder(args: &[&str]) -> Output { + Command::new(env!("CARGO_BIN_EXE_komorebi-recorder")) + .args(args) + .output() + .expect("run komorebi-recorder") +} + +fn json(args: &[&str]) -> Value { + let output = recorder(args); + assert!( + output.status.success(), + "{args:?} failed: {}", + String::from_utf8_lossy(&output.stderr) + ); + serde_json::from_slice(&output.stdout).expect("JSON") +} + +/// Display 1: 3 s at 30 fps that changes for a second, holds still for a second, then changes for +/// a second. Display 2: a 2 s video that only starts 1 s in. An event lands in the second change. +fn session(name: &str) -> Option { + if !testkit::ffmpeg_available() { + eprintln!("skipped: ffmpeg is not available to make a test video"); + return None; + } + + let (session, _) = SessionBuilder::new(name) + .video_showing(1, 0.0, 90, 30, |frame| match frame { + 0..=29 => frame, + 30..=59 => 29, + _ => frame, + }) + .video(2, 1.0, 60, 30) + .event( + 2.05, + json!({"type": "FocusChange", "content": [{"type": "Accessibility", "content": "AXApplicationActivated"}, 5, 1]}), + None, + ) + .build(); + Some(session.dir.display().to_string()) +} + +#[test] +fn the_picture_changing_is_found_where_it_changed_and_not_where_it_held_still() { + let Some(dir) = session("video-changes") else { + return; + }; + + let answer = json(&["changes", &dir, "--display", "1", "--json"]); + assert_eq!(answer["schema"], "changes/1"); + let spans = answer["data"]["spans"].as_array().expect("spans"); + assert_eq!(spans.len(), 2, "{spans:#?}"); + + let first = &spans[0]; + assert!(first["start_video_s"].as_f64().expect("start") < 0.1); + assert!((first["end_video_s"].as_f64().expect("end") - 0.97).abs() < 0.06); + let second = &spans[1]; + assert!((second["start_video_s"].as_f64().expect("start") - 2.0).abs() < 0.06); + assert!((second["end_video_s"].as_f64().expect("end") - 2.97).abs() < 0.06); + + // the event that landed in the second change is named there and not in the first + assert_eq!(second["events"], json!([1])); + assert_eq!(first["events"], json!([])); + + // counting up changes the low bits, which are the blocks on the right of the picture, top to + // bottom + let bbox = second["bbox"].as_array().expect("bbox"); + assert!(bbox[0].as_f64().expect("x") >= 0.5, "{bbox:?}"); + assert!(bbox[2].as_f64().expect("width") > 0.3, "{bbox:?}"); + assert!(bbox[3].as_f64().expect("height") > 0.9, "{bbox:?}"); + assert!(second["bbox_points"].is_array()); + assert!(second["peak_share"].as_f64().expect("share") > 0.0); +} + +#[test] +fn a_display_with_no_video_for_part_of_the_range_has_a_gap() { + let Some(dir) = session("video-gaps") else { + return; + }; + + let answer = json(&["changes", &dir, "--from", "0", "--to", "4", "--json"]); + let gaps = answer["data"]["gaps"].as_array().expect("gaps"); + let for_two: Vec<(f64, f64)> = gaps + .iter() + .filter(|gap| gap["display"] == 2) + .map(|gap| { + ( + gap["from_video_s"].as_f64().expect("from"), + gap["to_video_s"].as_f64().expect("to"), + ) + }) + .collect(); + assert_eq!(for_two.len(), 2, "{gaps:#?}"); + assert!((for_two[0].0 - 0.0).abs() < 1e-6 && (for_two[0].1 - 1.0).abs() < 1e-6); + assert!((for_two[1].0 - 3.0).abs() < 1e-6 && (for_two[1].1 - 4.0).abs() < 1e-6); + + // display 1 covers the first three seconds, so it only has a gap after them + assert!(gaps.iter().filter(|gap| gap["display"] == 1).count() <= 1); + + assert_eq!( + recorder(&["changes", &dir, "--display", "9"]).status.code(), + Some(3) + ); + assert_eq!( + recorder(&["changes", &dir, "--sensitivity", "0"]) + .status + .code(), + Some(2) + ); +} + +/// Read what the sheet shows in a cell back out of the PNG as a frame number. +fn frame_shown_in(png_path: &str, frame: &Value) -> u32 { + let decoder = png::Decoder::new(std::io::BufReader::new( + std::fs::File::open(png_path).expect("png"), + )); + let mut reader = decoder.read_info().expect("info"); + let mut pixels = vec![0; reader.output_buffer_size().expect("size")]; + let info = reader.next_frame(&mut pixels).expect("frame"); + + let (x, y) = ( + frame["x"].as_u64().expect("x") as usize, + frame["y"].as_u64().expect("y") as usize, + ); + let (width, height) = ( + frame["width"].as_u64().expect("width") as usize, + frame["height"].as_u64().expect("height") as usize, + ); + let mut bgra = Vec::with_capacity(width * height * 4); + for row in 0..height { + for column in 0..width { + let at = ((y + row) * info.width as usize + x + column) * 4; + bgra.extend_from_slice(&[pixels[at + 2], pixels[at + 1], pixels[at], 255]); + } + } + + testkit::numbered_frame_index(&bgra, width as u32, height as u32) +} + +#[test] +fn a_sheet_shows_the_frame_that_was_on_screen_at_each_moment() { + let Some(dir) = session("video-sheet") else { + return; + }; + + let answer = json(&[ + "sheet", + &dir, + "--display", + "1", + "--at", + "0.5", + "--at", + "1.5", + "--at", + "2.5", + "--width", + "128", + "--json", + ]); + assert_eq!(answer["schema"], "sheet/1"); + let path = answer["data"]["path"].as_str().expect("path"); + assert!(path.ends_with(".png") && std::path::Path::new(path).is_file()); + + let frames = answer["data"]["frames"].as_array().expect("frames"); + assert_eq!(frames.len(), 3); + // 0.5 s is frame 15; the picture then holds on frame 29's until 2.0 s; 2.5 s is frame 75 + assert_eq!(frame_shown_in(path, &frames[0]), 15); + assert_eq!(frame_shown_in(path, &frames[1]), 29); + assert_eq!(frame_shown_in(path, &frames[2]), 75); + + // each frame says when it really was on screen, never later than asked + for frame in frames { + let shown = frame["shown_video_s"].as_f64().expect("shown"); + let asked = frame["requested_video_s"].as_f64().expect("asked"); + assert!(shown <= asked + 1e-6 && asked - shown < 1.1, "{frame}"); + } +} + +#[test] +fn a_sheet_marks_the_bug_region_and_names_the_events_around_a_frame() { + let Some(dir) = session("video-sheet-bug") else { + return; + }; + let bug = json(&[ + "annotate", + &dir, + "add", + "--kind", + "bug", + "--start", + "2", + "--end", + "3", + "--text", + "the layout jumped", + "--json", + ]); + let id = bug["data"]["annotation"]["id"] + .as_str() + .expect("id") + .to_string(); + + let answer = json(&[ + "sheet", + &dir, + "--display", + "1", + "--bug", + &id, + "--count", + "4", + "--json", + ]); + let frames = answer["data"]["frames"].as_array().expect("frames"); + assert_eq!(frames.len(), 4); + + // the range runs from before the region to its end; frames inside it are marked + let marked: Vec = frames + .iter() + .map(|frame| !frame["bugs"].as_array().expect("bugs").is_empty()) + .collect(); + assert_eq!(marked, [false, false, true, true]); + assert_eq!(frames[3]["bugs"], json!([id])); + + // the focus change at 2.05 s is near the frame at 2.0 s + let near: Vec<&Value> = frames + .iter() + .filter(|frame| frame["events"] == json!([1])) + .collect(); + assert_eq!(near.len(), 1, "{frames:#?}"); +} + +#[test] +fn a_moment_a_display_had_no_video_is_skipped_not_invented() { + let Some(dir) = session("video-sheet-gap") else { + return; + }; + + let answer = json(&[ + "sheet", + &dir, + "--display", + "2", + "--at", + "0.5", + "--at", + "1.5", + "--json", + ]); + assert_eq!( + answer["data"]["frames"].as_array().expect("frames").len(), + 1 + ); + let skipped = answer["data"]["skipped"].as_array().expect("skipped"); + assert_eq!(skipped.len(), 1); + assert_eq!(skipped[0]["display"], 2); + + // nothing to show at all is not found + assert_eq!( + recorder(&["sheet", &dir, "--display", "2", "--at", "0.5"]) + .status + .code(), + Some(3) + ); + // a sheet that would be absurdly large is refused with how to fix it + let huge = recorder(&[ + "sheet", + &dir, + "--count", + "400", + "--columns", + "40", + "--width", + "1000", + ]); + assert_eq!(huge.status.code(), Some(2)); +}