diff --git a/docs/usage/investigating-a-recording.md b/docs/usage/investigating-a-recording.md index 26905c02..b38e45f9 100644 --- a/docs/usage/investigating-a-recording.md +++ b/docs/usage/investigating-a-recording.md @@ -119,6 +119,10 @@ and `event:12.end` when it finished. | `search TEXT` | Where a word appears: komorebi's log, events, incidents, annotations (and `--oslog` for the macOS trace) | | `changes` | When and where on each display the picture changed, and which events fall inside | | `sheet` | A contact sheet image of frames with times, events and the bug region marked | +| `motion [--region x,y,w,h]` | When, and how hard, a display or a region of it was moving: segments with start, duration, frame count, peak share of pixels changed and peak mean brightness delta (replaces scripting ffmpeg `tblend`) | +| `framediff A B [--region ..] [--png FILE] [--against SESSION]` | How two frames differ: changed share, mean and peak brightness delta, the box of change, and a PNG with the changed pixels in red | +| `blank --region x,y,w,h` | When a region was blank (one colour, named: `white`, `black`, a hex code) or frozen (identical to the frame before): "the window went white for 200 ms" | +| `compare A B` | Two recordings side by side: duration, frames, effective fps, worst frame gap, motion, findings by kind, annotations, and the difference | | `annotate`, `annotations` | Add, change and list bug regions, notes and findings | ### Who changed it diff --git a/komorebi-recorder/src/analysis/blank.rs b/komorebi-recorder/src/analysis/blank.rs new file mode 100644 index 00000000..01e5b556 --- /dev/null +++ b/komorebi-recorder/src/analysis/blank.rs @@ -0,0 +1,515 @@ +//! `blank`: when a region of a display was a blank patch of one colour, or frozen. +//! +//! The question behind it is "did the content of this window go white for a moment?". Every frame +//! of the range is cut to the region and measured, and two kinds of stretch are reported: +//! +//! - **blank**: the region is near-uniform, its brightness varying less than a threshold +//! (variance of the 0 to 255 brightness, default 4.0: a standard deviation of 2), and the colour +//! is named (`white`, `black`, `gray`, or a hex code). A run of blank frames of one colour is one +//! stretch, which lasts until the next frame replaced it. +//! - **frozen**: consecutive frames in which the region is identical (no pixel differs by more than +//! 2 of 255). The video only has a frame when something on the screen changed, so a frozen +//! region is one that stayed the same while something else on the display kept changing: a +//! stretch from the frame that showed it to the last repeat. A display that held entirely +//! still has no frames and so no frozen stretch. +//! +//! Stretches shorter than `min_duration_s` are dropped. A stretch is judged on frames decoded at a +//! size that gives the region 240 pixels across, so a few stray pixels in a large region are +//! averaged away; use a smaller region to see them. + +use super::Analysis; +use super::changes::Gap; +use super::envelope; +use super::motion::ScanOptions; +use super::motion::scale_for; +use super::pixels; +use super::pixels::Crop; +use super::video::SessionVideo; +use crate::av::Scale; +use color_eyre::eyre; +use schemars::JsonSchema; +use serde::Serialize; + +/// The schema name that identifies a [`BlankData`] in an envelope (`blank/1`). +pub const SCHEMA_BLANK: &str = "blank/1"; + +/// The default brightness variance under which a region is blank. +pub const DEFAULT_VARIANCE: f64 = 4.0; + +/// The default shortest stretch reported, in seconds. +pub const DEFAULT_MIN_DURATION_S: f64 = 0.0; + +/// A pixel that moved by no more than this (of 255) is the same pixel. +const FROZEN_TOLERANCE: i32 = 2; + +/// What a [`Stretch`] is. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, JsonSchema)] +#[serde(rename_all = "lowercase")] +pub enum StretchKind { + /// The region was one colour. + Blank, + /// The region was identical in consecutive frames. + Frozen, +} + +/// A stretch of video in which the region was blank, or frozen. +#[derive(Debug, Clone, PartialEq, Serialize, JsonSchema)] +pub struct Stretch { + /// The id of the display. + pub display: u32, + /// Whether the region was blank or frozen. + pub kind: StretchKind, + /// When the stretch began, in seconds of video time. + pub start_video_s: f64, + /// When it ended, in seconds of video time: when a frame showed something else (a blank stretch + /// that lasted to the end of the range ends there), or the last repeat for a frozen one. + pub end_video_s: f64, + /// `end_video_s - start_video_s`. + pub duration_s: f64, + /// For a blank stretch the frames that were blank; for a frozen one the frames that repeated + /// the one before. + pub frames: usize, + /// For a blank stretch, the colour: `white`, `black`, `gray` or a hex code. `None` when frozen. + pub colour: Option, + /// For a blank stretch, the mean colour `[red, green, blue]`. `None` when frozen. + pub rgb: Option<[u8; 3]>, +} + +/// Where a region was blank or frozen over a range: the `data` of `blank`. +#[derive(Debug, Clone, PartialEq, Serialize, JsonSchema)] +pub struct BlankData { + /// The start of the range scanned, in seconds of video time. + pub from_video_s: f64, + /// The end of the range scanned, in seconds of video time. + pub to_video_s: f64, + /// The displays scanned. + pub displays: Vec, + /// The region, in global display points; `None` is the whole display. + pub region: Option, + /// A region is blank when the variance of its brightness is below this. + pub variance_threshold: f64, + /// Stretches shorter than this were dropped, in seconds. + pub min_duration_s: f64, + /// How many frames were decoded and measured, over all displays. + pub frames_examined: usize, + /// The stretches, ordered by start, then display, then kind. + pub stretches: Vec, + /// Stretches of the range where a display had no video at all. + pub gaps: Vec, +} + +impl BlankData { + /// Renders the answer as text, one line for each stretch. + /// + /// # Example + /// + /// ```rust + /// use komorebi_recorder::analysis::blank::{BlankData, Stretch, StretchKind}; + /// + /// let data = BlankData { + /// from_video_s: 0.0, + /// to_video_s: 6.0, + /// displays: vec![1], + /// region: None, + /// variance_threshold: 4.0, + /// min_duration_s: 0.0, + /// frames_examined: 50, + /// stretches: vec![Stretch { + /// display: 1, + /// kind: StretchKind::Blank, + /// start_video_s: 3.28, + /// end_video_s: 3.5, + /// duration_s: 0.22, + /// frames: 3, + /// colour: Some("white".to_string()), + /// rgb: Some([255, 255, 255]), + /// }], + /// gaps: vec![], + /// }; + /// assert!(data.to_text().contains("blank (white)")); + /// ``` + #[must_use] + pub fn to_text(&self) -> String { + use crate::timecode::format as time; + use std::fmt::Write; + + let mut out = String::new(); + let _ = writeln!( + out, + "{} frames measured on display(s) {} from {} to {}: {} stretch(es) blank or frozen", + self.frames_examined, + self.displays + .iter() + .map(ToString::to_string) + .collect::>() + .join(", "), + time(self.from_video_s), + time(self.to_video_s), + self.stretches.len(), + ); + for stretch in &self.stretches { + let what = match (&stretch.kind, &stretch.colour) { + (StretchKind::Blank, Some(colour)) => format!("blank ({colour})"), + (StretchKind::Blank, None) => "blank".to_string(), + (StretchKind::Frozen, _) => "frozen".to_string(), + }; + let _ = writeln!( + out, + "display {} {} - {} ({:.0} ms, {} frames) {what}", + stretch.display, + time(stretch.start_video_s), + time(stretch.end_video_s), + stretch.duration_s * 1000.0, + stretch.frames, + ); + } + for gap in &self.gaps { + let _ = writeln!( + out, + "display {} has no video from {} to {}", + gap.display, + time(gap.from_video_s), + time(gap.to_video_s) + ); + } + out + } +} + +/// What one frame showed in the region. +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct Sample { + /// When the frame was shown, in seconds of video time. + pub shown_s: f64, + /// The variance of the region's brightness. + pub variance: f64, + /// The mean colour of the region. + pub rgb: [u8; 3], + /// Whether the region is identical to what the previous frame showed. + pub identical: bool, +} + +/// Finds the blank and frozen stretches in the samples of one display, which are in time order. +/// `to_s` is where the range ends: a blank stretch still going at the last sample lasts to it. +/// +/// # Example +/// +/// ```rust +/// use komorebi_recorder::analysis::blank::{Sample, StretchKind, stretches}; +/// +/// let white = |shown_s, identical| Sample { shown_s, variance: 0.0, rgb: [255, 255, 255], identical }; +/// let busy = |shown_s| Sample { shown_s, variance: 900.0, rgb: [90, 80, 70], identical: false }; +/// +/// // busy until 1.0 s, white from 1.0 s to 1.4 s, busy again +/// let found = stretches(1, &[busy(0.5), white(1.0, false), white(1.2, true), busy(1.4)], 2.0, 4.0, 0.0); +/// assert_eq!(found.len(), 2); +/// assert_eq!(found[0].kind, StretchKind::Blank); +/// assert_eq!(found[0].colour.as_deref(), Some("white")); +/// assert!((found[0].duration_s - 0.4).abs() < 1e-9); +/// assert_eq!(found[1].kind, StretchKind::Frozen); +/// ``` +#[must_use] +pub fn stretches( + display: u32, + samples: &[Sample], + to_s: f64, + variance: f64, + min_duration_s: f64, +) -> Vec { + let mut found = Vec::new(); + + // blank: runs of blank frames of one colour + let mut index = 0; + while index < samples.len() { + let sample = &samples[index]; + if sample.variance >= variance { + index += 1; + continue; + } + + let name = pixels::colour_name(sample.rgb); + let mut last = index; + while samples + .get(last + 1) + .is_some_and(|next| next.variance < variance && pixels::colour_name(next.rgb) == name) + { + last += 1; + } + + let run = &samples[index..=last]; + let end = samples + .get(last + 1) + .map_or(to_s.max(samples[last].shown_s), |next| next.shown_s); + let mean = |channel: usize| { + (run.iter().map(|s| f64::from(s.rgb[channel])).sum::() / run.len() as f64).round() + as u8 + }; + found.push(Stretch { + display, + kind: StretchKind::Blank, + start_video_s: sample.shown_s, + end_video_s: end, + duration_s: end - sample.shown_s, + frames: run.len(), + colour: Some(name), + rgb: Some([mean(0), mean(1), mean(2)]), + }); + index = last + 1; + } + + // frozen: runs of frames identical to the one before, from that one + let mut index = 1; + while index < samples.len() { + if !samples[index].identical { + index += 1; + continue; + } + + let mut last = index; + while samples.get(last + 1).is_some_and(|next| next.identical) { + last += 1; + } + let (start, end) = (samples[index - 1].shown_s, samples[last].shown_s); + found.push(Stretch { + display, + kind: StretchKind::Frozen, + start_video_s: start, + end_video_s: end, + duration_s: end - start, + frames: last - index + 1, + colour: None, + rgb: None, + }); + index = last + 1; + } + + found.retain(|stretch| stretch.duration_s >= min_duration_s); + found.sort_by(|a, b| { + a.start_video_s + .total_cmp(&b.start_video_s) + .then((a.kind as u8).cmp(&(b.kind as u8))) + }); + found +} + +impl Analysis { + /// Finds where the region of `options` (the whole display if it has none) was blank, or + /// frozen, on each display, as the [module](self) describes. + /// + /// `variance` is the brightness variance under which a region is blank (see + /// [`DEFAULT_VARIANCE`]); stretches shorter than `min_duration_s` are dropped. Stops early, + /// without an error, once `options.cancel` is set. + /// + /// # Errors + /// + /// Returns an error if `variance` is negative or `min_duration_s` is, or the region does not + /// overlap a display or a display's frame is unknown (the usage kind), or a video file cannot + /// be resolved, opened or decoded, or the host has no decoder. + /// + /// # Example + /// + /// This needs recorded video, so it is not run here; `tests/video.rs` runs it through the + /// `blank` command. + /// + /// ```rust,no_run + /// use komorebi_recorder::analysis::Analysis; + /// use komorebi_recorder::analysis::blank::{DEFAULT_MIN_DURATION_S, DEFAULT_VARIANCE}; + /// use komorebi_recorder::analysis::motion::ScanOptions; + /// use komorebi_recorder::analysis::pixels::Crop; + /// use komorebi_recorder::analysis::video::SessionVideo; + /// + /// let analysis = Analysis::open(std::path::Path::new("session-dir"))?; + /// let video = SessionVideo::new(&analysis.manifest, analysis.guard()); + /// let data = analysis.blank( + /// &video, + /// &ScanOptions { + /// displays: &video.displays(), + /// from_s: 0.0, + /// to_s: 60.0, + /// crop: Some(Crop::parse("500,300,1200,800")?), + /// cancel: None, + /// }, + /// DEFAULT_VARIANCE, + /// DEFAULT_MIN_DURATION_S, + /// )?; + /// print!("{}", data.to_text()); + /// # Ok::<(), Box>(()) + /// ``` + pub fn blank( + &self, + video: &SessionVideo, + options: &ScanOptions, + variance: f64, + min_duration_s: f64, + ) -> eyre::Result { + if variance < 0.0 || min_duration_s < 0.0 { + return Err(envelope::usage( + "--variance and --min-ms cannot be negative", + )); + } + + let mut found = Vec::new(); + let mut examined = 0; + for display in options.displays { + let (crop, scale) = match &options.crop { + Some(crop) => { + let fractions = self.crop_fractions(*display, crop)?; + (Some(fractions), scale_for(fractions[2])) + } + None => (None, Scale::Width(240)), + }; + + let mut samples: Vec = Vec::new(); + let mut previous: Option<(Vec, u32, u32)> = None; + examined += video.scan( + *display, + options.from_s, + options.to_s, + scale, + &mut |decoded| { + if options + .cancel + .is_some_and(|cancel| cancel.load(std::sync::atomic::Ordering::Relaxed)) + { + return false; + } + + let (width, height) = (decoded.frame.width, decoded.frame.height); + let rect = crop.map(|crop| pixels::pixel_rect(crop, width, height)); + let (patch, row) = match rect { + Some(rect) => ( + pixels::crop_bgra(&decoded.frame.bgra, width, rect), + rect.width, + ), + None => (decoded.frame.bgra.clone(), width), + }; + let now = pixels::luma(&patch); + let identical = previous.as_ref().is_some_and(|(before, w, h)| { + (*w, *h) == (width, height) + && pixels::delta(before, &now, row, FROZEN_TOLERANCE).changed == 0 + }); + let measured = pixels::evenness(&patch); + samples.push(Sample { + shown_s: decoded.shown_s, + variance: measured.variance, + rgb: measured.rgb, + identical, + }); + previous = Some((now, width, height)); + true + }, + )?; + + found.extend(stretches( + *display, + &samples, + options.to_s, + variance, + min_duration_s, + )); + } + + found.sort_by(|a, b| { + a.start_video_s + .total_cmp(&b.start_video_s) + .then(a.display.cmp(&b.display)) + }); + + Ok(BlankData { + from_video_s: options.from_s, + to_video_s: options.to_s, + displays: options.displays.to_vec(), + region: options.crop, + variance_threshold: variance, + min_duration_s, + frames_examined: examined, + stretches: found, + gaps: self.video_gaps(options.displays, options.from_s, options.to_s), + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn sample(shown_s: f64, variance: f64, rgb: [u8; 3], identical: bool) -> Sample { + Sample { + shown_s, + variance, + rgb, + identical, + } + } + + #[test] + fn a_blank_stretch_lasts_until_the_frame_that_replaced_it() { + let samples = [ + sample(0.0, 800.0, [90, 90, 90], false), + sample(1.0, 0.5, [250, 250, 250], false), + sample(1.5, 0.5, [250, 250, 250], false), + sample(2.0, 800.0, [90, 90, 90], false), + ]; + let found = stretches(7, &samples, 5.0, 4.0, 0.0); + assert_eq!(found.len(), 1); + assert_eq!((found[0].start_video_s, found[0].end_video_s), (1.0, 2.0)); + assert_eq!(found[0].frames, 2); + assert_eq!(found[0].display, 7); + } + + #[test] + fn a_blank_stretch_at_the_end_lasts_to_the_end_of_the_range() { + let samples = [ + sample(0.0, 800.0, [9; 3], false), + sample(4.0, 0.0, [0; 3], false), + ]; + let found = stretches(1, &samples, 6.0, 4.0, 0.0); + assert_eq!(found[0].colour.as_deref(), Some("black")); + assert_eq!(found[0].end_video_s, 6.0); + } + + #[test] + fn a_change_of_blank_colour_is_two_stretches() { + let samples = [ + sample(0.0, 0.0, [255; 3], false), + sample(1.0, 0.0, [0; 3], false), + ]; + let found = stretches(1, &samples, 2.0, 4.0, 0.0); + let colours: Vec<_> = found.iter().filter_map(|s| s.colour.as_deref()).collect(); + assert_eq!(colours, ["white", "black"]); + } + + #[test] + fn frozen_runs_start_at_the_frame_that_was_repeated() { + let samples = [ + sample(0.0, 800.0, [90; 3], false), + sample(0.1, 800.0, [90; 3], false), + sample(0.2, 800.0, [90; 3], true), + sample(0.3, 800.0, [90; 3], true), + sample(0.4, 800.0, [90; 3], false), + ]; + let found = stretches(1, &samples, 1.0, 4.0, 0.0); + assert_eq!(found.len(), 1); + assert_eq!(found[0].kind, StretchKind::Frozen); + assert_eq!((found[0].start_video_s, found[0].end_video_s), (0.1, 0.3)); + assert_eq!(found[0].frames, 2); + } + + #[test] + fn short_stretches_are_dropped() { + let samples = [ + sample(0.0, 0.0, [255; 3], false), + sample(0.05, 800.0, [90; 3], false), + ]; + assert!(stretches(1, &samples, 1.0, 4.0, 0.1).is_empty()); + assert_eq!(stretches(1, &samples, 1.0, 4.0, 0.0).len(), 1); + } + + #[test] + fn a_synthetic_white_patch_is_measured_as_blank_white() { + let white = vec![255_u8; 4 * 4 * 4]; + let measured = pixels::evenness(&white); + assert!(measured.variance < DEFAULT_VARIANCE); + assert_eq!(pixels::colour_name(measured.rgb), "white"); + } +} diff --git a/komorebi-recorder/src/analysis/changes.rs b/komorebi-recorder/src/analysis/changes.rs index d7c8ff53..cf88d7c2 100644 --- a/komorebi-recorder/src/analysis/changes.rs +++ b/komorebi-recorder/src/analysis/changes.rs @@ -37,7 +37,7 @@ use super::Analysis; use super::envelope; -use super::video::DecodedFrame; +use super::pixels; use super::video::SessionVideo; use crate::av::Scale; use color_eyre::eyre; @@ -52,10 +52,10 @@ pub const SCHEMA_CHANGES: &str = "changes/1"; const COMPARE_WIDTH: u32 = 240; /// A pixel counts as changed if its brightness moved by more than this (out of 255). -const PIXEL_THRESHOLD: i32 = 24; +pub(super) const PIXEL_THRESHOLD: i32 = 24; /// A frame is a change if at least this share of its pixels changed, at sensitivity 1. -const SHARE_THRESHOLD: f64 = 0.0005; +pub(super) const SHARE_THRESHOLD: f64 = 0.0005; /// Frames of one change closer together than this are one change. const MERGE_GAP_S: f64 = 0.15; @@ -127,28 +127,176 @@ pub struct ChangesData { pub gaps: Vec, } -/// The brightness of every pixel of a BGRA frame. -fn luma(frame: &DecodedFrame) -> Vec { - frame - .frame - .bgra - .as_chunks::<4>() - .0 - .iter() - .map(|pixel| { - // Rec. 601 weights in integers - ((u32::from(pixel[2]) * 299 + u32::from(pixel[1]) * 587 + u32::from(pixel[0]) * 114) - / 1000) as u8 - }) - .collect() +/// One stretch of change being built: shared by `changes` and `motion`. +pub(super) struct Open { + pub(super) start: f64, + pub(super) end: f64, + pub(super) frames: usize, + pub(super) peak: f64, + /// The largest mean brightness difference between two consecutive frames in the stretch. + pub(super) peak_delta: f64, + /// `(min_x, min_y, max_x, max_y)` in pixels of the decoded frame. + pub(super) bounds: (u32, u32, u32, u32), } -struct Open { - start: f64, - end: f64, - frames: usize, - peak: f64, +/// Groups changed frames into stretches: frames less than [`MERGE_GAP_S`] apart are one. +#[derive(Default)] +pub(super) struct Segments { + open: Option, + finished: Vec, +} + +impl Segments { + /// Adds a changed frame shown at `shown_s`, `share` of whose compared pixels changed, whose + /// mean brightness difference from the frame before was `delta` and whose changed pixels lie + /// in `bounds`. + pub(super) fn push( + &mut self, + shown_s: f64, + share: f64, + delta: f64, + bounds: (u32, u32, u32, u32), + ) { + match &mut self.open { + Some(span) if shown_s - span.end <= MERGE_GAP_S => { + span.end = shown_s; + span.frames += 1; + span.peak = span.peak.max(share); + span.peak_delta = span.peak_delta.max(delta); + span.bounds = ( + span.bounds.0.min(bounds.0), + span.bounds.1.min(bounds.1), + span.bounds.2.max(bounds.2), + span.bounds.3.max(bounds.3), + ); + } + _ => { + self.finished.extend(self.open.take()); + self.open = Some(Open { + start: shown_s, + end: shown_s, + frames: 1, + peak: share, + peak_delta: delta, + bounds, + }); + } + } + } + + /// The stretches, in order, including the one still open. + pub(super) fn finish(mut self) -> Vec { + self.finished.extend(self.open.take()); + self.finished + } +} + +/// How [`track`] looks for change. +pub(super) struct Tracking<'a> { + /// The share of the compared pixels that must change for a frame to count. + pub(super) threshold: f64, + /// Only compare this part of the frame, as fractions `[x, y, width, height]` of it. + pub(super) crop: Option<[f64; 4]>, + /// What size to decode at. + pub(super) scale: Scale, + /// Stop early once this is set. + pub(super) cancel: Option<&'a std::sync::atomic::AtomicBool>, +} + +/// What [`track`] found on one display. +pub(super) struct Tracked { + pub(super) spans: Vec, + /// The size of the last frame decoded. + pub(super) dimensions: Option<(u32, u32)>, + pub(super) examined: usize, +} + +/// Decodes `display` over `from_s..=to_s` and finds the stretches in which it changed: the engine +/// of `changes` and `motion`, described in the [module](self) documentation. +pub(super) fn track( + video: &SessionVideo, + display: u32, + from_s: f64, + to_s: f64, + how: &Tracking, +) -> eyre::Result { + let mut previous: Option<(Vec, u32, u32)> = None; + let mut segments = Segments::default(); + + let examined = video.scan(display, from_s, to_s, how.scale, &mut |decoded| { + if how + .cancel + .is_some_and(|cancel| cancel.load(std::sync::atomic::Ordering::Relaxed)) + { + return false; + } + let (width, height) = (decoded.frame.width, decoded.frame.height); + let whole = pixels::luma(&decoded.frame.bgra); + let rect = how.crop.map(|crop| pixels::pixel_rect(crop, width, height)); + let now = match rect { + Some(rect) => pixels::crop_luma(&whole, width, rect), + None => whole, + }; + let (row, origin) = rect.map_or((width, (0, 0)), |rect| (rect.width, (rect.x, rect.y))); + + if let Some((before, before_width, before_height)) = &previous + && (*before_width, *before_height) == (width, height) + { + let found = pixels::delta(before, &now, row, PIXEL_THRESHOLD); + let share = found.share(); + if share >= how.threshold { + let (min_x, min_y, max_x, max_y) = found.bounds.unwrap_or((0, 0, 0, 0)); + let (min_x, min_y, max_x, max_y) = ( + min_x + origin.0, + min_y + origin.1, + max_x + origin.0, + max_y + origin.1, + ); + segments.push( + decoded.shown_s, + share, + found.mean, + (min_x, min_y, max_x, max_y), + ); + } + } + + previous = Some((now, width, height)); + true + })?; + + Ok(Tracked { + spans: segments.finish(), + dimensions: previous.map(|(_, width, height)| (width, height)), + examined, + }) +} + +/// The box `bounds` (pixels) as fractions of a `dimensions` frame: `[x, y, width, height]`. +pub(super) fn fractions_of( bounds: (u32, u32, u32, u32), + dimensions: Option<(u32, u32)>, +) -> [f64; 4] { + let (width, height) = dimensions.unwrap_or((1, 1)); + let (w, h) = (f64::from(width), f64::from(height)); + [ + f64::from(bounds.0) / w, + f64::from(bounds.1) / h, + f64::from(bounds.2 - bounds.0 + 1) / w, + f64::from(bounds.3 - bounds.1 + 1) / h, + ] +} + +/// Fractions of a display's frame `[x, y, width, height]` in global points. +pub(super) fn points_of(frame_box: Option<[f64; 4]>, bbox: [f64; 4]) -> Option<[f64; 4]> { + frame_box.map(|[x, y, frame_width, frame_height]| { + [ + x + bbox[0] * frame_width, + y + bbox[1] * frame_height, + bbox[2] * frame_width, + bbox[3] * frame_height, + ] + }) } impl Analysis { @@ -247,86 +395,22 @@ impl Analysis { .find(|entry| entry.result.display.id == *display) .map(|entry| entry.result.display.frame); - let mut previous: Option<(Vec, u32, u32)> = None; - let mut open: Option = None; - let mut finished: Vec = Vec::new(); - - examined += video.scan( + let tracked = track( + video, *display, from_s, to_s, - Scale::Width(COMPARE_WIDTH), - &mut |decoded| { - if cancel - .is_some_and(|cancel| cancel.load(std::sync::atomic::Ordering::Relaxed)) - { - return false; - } - let now = luma(decoded); - let (width, height) = (decoded.frame.width, decoded.frame.height); - - if let Some((before, before_width, before_height)) = &previous - && (*before_width, *before_height) == (width, height) - { - let (mut count, mut min_x, mut min_y, mut max_x, mut max_y) = - (0_usize, u32::MAX, u32::MAX, 0_u32, 0_u32); - for (index, (a, b)) in before.iter().zip(&now).enumerate() { - if (i32::from(*a) - i32::from(*b)).abs() > PIXEL_THRESHOLD { - let (x, y) = (index as u32 % width, index as u32 / width); - count += 1; - min_x = min_x.min(x); - max_x = max_x.max(x); - min_y = min_y.min(y); - max_y = max_y.max(y); - } - } - - let share = count as f64 / now.len().max(1) as f64; - if share >= threshold { - match &mut open { - Some(span) if decoded.shown_s - span.end <= MERGE_GAP_S => { - span.end = decoded.shown_s; - span.frames += 1; - span.peak = span.peak.max(share); - span.bounds = ( - span.bounds.0.min(min_x), - span.bounds.1.min(min_y), - span.bounds.2.max(max_x), - span.bounds.3.max(max_y), - ); - } - _ => { - finished.extend(open.take()); - open = Some(Open { - start: decoded.shown_s, - end: decoded.shown_s, - frames: 1, - peak: share, - bounds: (min_x, min_y, max_x, max_y), - }); - } - } - } - } - - previous = Some((now, width, height)); - true + &Tracking { + threshold, + crop: None, + scale: Scale::Width(COMPARE_WIDTH), + cancel, }, )?; - finished.extend(open.take()); + examined += tracked.examined; - let dimensions = previous - .as_ref() - .map(|(_, width, height)| (*width, *height)); - for span in finished { - let (width, height) = dimensions.unwrap_or((1, 1)); - let (w, h) = (f64::from(width), f64::from(height)); - let bbox = [ - f64::from(span.bounds.0) / w, - f64::from(span.bounds.1) / h, - f64::from(span.bounds.2 - span.bounds.0 + 1) / w, - f64::from(span.bounds.3 - span.bounds.1 + 1) / h, - ]; + for span in tracked.spans { + let bbox = fractions_of(span.bounds, tracked.dimensions); let events = self .events() .iter() @@ -344,14 +428,7 @@ impl Analysis { frames: span.frames, peak_share: span.peak, bbox, - bbox_points: frame_box.map(|[x, y, frame_width, frame_height]| { - [ - x + bbox[0] * frame_width, - y + bbox[1] * frame_height, - bbox[2] * frame_width, - bbox[3] * frame_height, - ] - }), + bbox_points: points_of(frame_box, bbox), events, }); } @@ -442,3 +519,37 @@ impl Analysis { gaps } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn frames_close_together_are_one_stretch_and_a_pause_starts_another() { + let mut segments = Segments::default(); + segments.push(1.0, 0.10, 5.0, (4, 4, 5, 5)); + segments.push(1.1, 0.20, 9.0, (2, 6, 3, 8)); // 0.1 s later: the same stretch + segments.push(2.0, 0.05, 1.0, (0, 0, 0, 0)); // a second later: another + + let found = segments.finish(); + assert_eq!(found.len(), 2); + assert_eq!( + (found[0].start, found[0].end, found[0].frames), + (1.0, 1.1, 2) + ); + assert_eq!((found[0].peak, found[0].peak_delta), (0.20, 9.0)); + assert_eq!(found[0].bounds, (2, 4, 5, 8)); + assert_eq!((found[1].start, found[1].frames), (2.0, 1)); + } + + #[test] + fn a_box_is_fractions_of_the_frame_and_then_points_of_the_display() { + let bbox = fractions_of((10, 5, 19, 9), Some((100, 50))); + assert_eq!(bbox, [0.1, 0.1, 0.1, 0.1]); + assert_eq!( + points_of(Some([1000.0, 0.0, 2000.0, 1000.0]), bbox), + Some([1200.0, 100.0, 200.0, 100.0]) + ); + assert_eq!(points_of(None, bbox), None); + } +} diff --git a/komorebi-recorder/src/analysis/compare.rs b/komorebi-recorder/src/analysis/compare.rs new file mode 100644 index 00000000..5f2ace9c --- /dev/null +++ b/komorebi-recorder/src/analysis/compare.rs @@ -0,0 +1,520 @@ +//! `compare`: two recordings side by side, as numbers. +//! +//! For a before-and-after of a fix, or a good run against a bad one: how long each recording is, +//! how many frames it has and how steadily they came, how much the screen moved, what the analysis +//! found wrong, and how many annotations it carries, with the difference (`b` minus `a`) of each. +//! +//! | Measure | Where it comes from | +//! |---|---| +//! | `duration_s`, `frames`, `effective_fps` | the `frames-*.csv` logs of every display segment: real frames, and the intervals between them over the time they span | +//! | `median_gap_ms`, `worst_gap_ms`, `late_gaps` | the gaps between consecutive real frames of a segment; `late_gaps` counts those over 100 ms and over twice the median | +//! | `motion` | [`Analysis::motion`] over every display at sensitivity 1: segments, their total time, the largest share and mean brightness delta | +//! | `findings` | [`Analysis::findings`], counted by kind | +//! | `annotations`, `bugs` | the session's annotations | +//! +//! Frames come when the screen changes, so a still screen has long gaps and a low effective +//! frame rate without anything being wrong: compare like with like. + +use super::Analysis; +use super::motion::ScanOptions; +use super::video::SessionVideo; +use crate::annotations; +use color_eyre::eyre; +use schemars::JsonSchema; +use serde::Serialize; +use std::collections::BTreeMap; +use std::sync::atomic::AtomicBool; + +/// The schema name that identifies a [`CompareData`] in an envelope (`compare/1`). +pub const SCHEMA_COMPARE: &str = "compare/1"; + +/// A gap between frames over this is late (and over twice the median). +const LATE_GAP_S: f64 = 0.1; + +/// How the pictures of a recording moved. +#[derive(Debug, Clone, PartialEq, Serialize, JsonSchema)] +pub struct MotionMetrics { + /// How many segments of motion, over all displays. + pub segments: usize, + /// Their total duration, in seconds. + pub active_s: f64, + /// The largest share of a display that changed between two frames. + pub peak_share: f64, + /// The largest mean brightness difference between two frames, 0 to 255. + pub peak_mean_delta: f64, +} + +/// The measures of one recording. +#[derive(Debug, Clone, PartialEq, Serialize, JsonSchema)] +pub struct Side { + /// The session directory. + pub session: String, + /// The length of the video, in seconds. + pub duration_s: f64, + /// Real frames over every display segment. + pub frames: usize, + /// Intervals between frames over the time they span, per second; `0` with fewer than two + /// frames. + pub effective_fps: f64, + /// The median gap between consecutive frames, in milliseconds. + pub median_gap_ms: f64, + /// The longest gap between consecutive frames, in milliseconds. + pub worst_gap_ms: f64, + /// When the longest gap began, in seconds of video time. + pub worst_gap_at_video_s: Option, + /// Gaps over 100 ms and over twice the median. + pub late_gaps: usize, + /// The motion measures; `None` if the video could not be decoded (see `notes`). + pub motion: Option, + /// How many findings of each kind. + pub findings: BTreeMap, + /// How many findings in all. + pub findings_total: usize, + /// How many annotations of any kind. + pub annotations: usize, + /// How many of them are bug regions. + pub bugs: usize, +} + +/// `b` minus `a` for each measure. A motion measure is `None` if either side has none. +#[derive(Debug, Clone, PartialEq, Serialize, JsonSchema)] +pub struct Deltas { + /// Duration, in seconds. + pub duration_s: f64, + /// Frames. + pub frames: i64, + /// Effective frames per second. + pub effective_fps: f64, + /// Median frame gap, in milliseconds. + pub median_gap_ms: f64, + /// Worst frame gap, in milliseconds. + pub worst_gap_ms: f64, + /// Late gaps. + pub late_gaps: i64, + /// Motion segments. + pub motion_segments: Option, + /// Total time in motion, in seconds. + pub active_s: Option, + /// Peak share of a display changed. + pub peak_share: Option, + /// Peak mean brightness delta. + pub peak_mean_delta: Option, + /// Findings in all. + pub findings_total: i64, + /// Findings by kind, for every kind either side has. + pub findings: BTreeMap, + /// Annotations. + pub annotations: i64, +} + +/// Two recordings side by side: the `data` of `compare`. +#[derive(Debug, Clone, PartialEq, Serialize, JsonSchema)] +pub struct CompareData { + /// The first recording. + pub a: Side, + /// The second recording. + pub b: Side, + /// `b` minus `a`. + pub deltas: Deltas, + /// What could not be measured, and why. + pub notes: Vec, +} + +/// The gaps between real frames in the text of a `frames-*.csv`, as `(time of the earlier frame +/// in ns, gap in ns)`, and how many frames there were. Rows that do not parse (a header, a torn +/// last row) and repeat rows (`-`) are ignored. +/// +/// # Example +/// +/// ```rust +/// use komorebi_recorder::analysis::compare::frame_gaps; +/// +/// let (gaps, frames) = frame_gaps("index,pts,received\n0,1000,1\n-,1500,1\n1,3000,1\n2,3500,1\n3,"); +/// assert_eq!(frames, 3); +/// assert_eq!(gaps, [(1000, 2000), (3000, 500)]); +/// ``` +#[must_use] +pub fn frame_gaps(csv: &str) -> (Vec<(u64, u64)>, usize) { + let mut gaps = Vec::new(); + let mut frames = 0; + let mut previous: Option = None; + for line in csv.lines() { + let mut fields = line.split(','); + let (Some(index), Some(pts), Some(_), None) = + (fields.next(), fields.next(), fields.next(), fields.next()) + else { + continue; + }; + if index.parse::().is_err() { + continue; + } + let Ok(pts) = pts.parse::() else { + continue; + }; + + frames += 1; + if let Some(before) = previous { + gaps.push((before, pts.saturating_sub(before))); + } + previous = Some(pts); + } + (gaps, frames) +} + +impl Analysis { + /// Measures this recording as one [`Side`] of a comparison, as the [module](self) describes. + /// + /// The frame logs are read from the session. If the video cannot be decoded (the host has no + /// decoder, or there is no video) the motion measures are `None` and the reason goes in + /// `notes`. `cancel` stops the motion scan early. + /// + /// # Errors + /// + /// Returns an error if the findings cannot be computed (a recorded state is missing or + /// unreadable) or the annotations cannot be read. + pub fn side(&self, cancel: Option<&AtomicBool>, notes: &mut Vec) -> eyre::Result { + let mut all_gaps: Vec<(u64, u64)> = Vec::new(); + let (mut frames, mut span_ns, mut intervals) = (0_usize, 0_u128, 0_usize); + for entry in &self.manifest.displays { + let path = self.session.path(&entry.result.frames_file); + let Ok(text) = std::fs::read_to_string(&path) else { + continue; + }; + let (gaps, count) = frame_gaps(&text); + frames += count; + intervals += gaps.len(); + span_ns += gaps.iter().map(|(_, gap)| u128::from(*gap)).sum::(); + all_gaps.extend(gaps); + } + + let mut sorted: Vec = all_gaps.iter().map(|(_, gap)| *gap).collect(); + sorted.sort_unstable(); + let median_ns = sorted.get(sorted.len() / 2).copied().unwrap_or(0); + let worst = all_gaps.iter().max_by_key(|(_, gap)| *gap).copied(); + let late_ns = (LATE_GAP_S * 1e9) as u64; + let late_gaps = all_gaps + .iter() + .filter(|(_, gap)| *gap > late_ns && *gap > median_ns.saturating_mul(2)) + .count(); + + let video = SessionVideo::new(&self.manifest, self.guard()); + let displays = video.displays(); + let (_, duration) = self.manifest.video_span(); + let motion = if displays.is_empty() { + notes.push(format!( + "{} recorded no video, so there are no motion measures", + self.session.dir.display() + )); + None + } else { + match self.motion( + &video, + &ScanOptions { + displays: &displays, + from_s: 0.0, + to_s: duration, + crop: None, + cancel, + }, + 1.0, + ) { + Ok(data) => Some(MotionMetrics { + segments: data.segments.len(), + active_s: data.active_s(), + peak_share: data + .segments + .iter() + .map(|segment| segment.peak_share) + .fold(0.0, f64::max), + peak_mean_delta: data + .segments + .iter() + .map(|segment| segment.peak_mean_delta) + .fold(0.0, f64::max), + }), + Err(error) => { + notes.push(format!( + "{}: motion could not be measured: {error}", + self.session.dir.display() + )); + None + } + } + }; + + let mut findings: BTreeMap = BTreeMap::new(); + let found = self.findings()?; + for finding in &found { + *findings.entry(finding.kind.clone()).or_default() += 1; + } + let loaded = annotations::load(&self.session.dir)?; + + Ok(Side { + session: self.session.dir.display().to_string(), + duration_s: duration, + frames, + effective_fps: if span_ns == 0 { + 0.0 + } else { + intervals as f64 / (span_ns as f64 / 1e9) + }, + median_gap_ms: median_ns as f64 / 1e6, + worst_gap_ms: worst.map_or(0.0, |(_, gap)| gap as f64 / 1e6), + worst_gap_at_video_s: worst.map(|(at, _)| self.video_s(at)), + late_gaps, + motion, + findings_total: found.len(), + findings, + annotations: loaded.annotations.len(), + bugs: loaded.bugs().count(), + }) + } + + /// Compares this recording (`a`) with `other` (`b`): see the [module](self) documentation. + /// + /// # Errors + /// + /// The same as [`Analysis::side`], for either recording. + /// + /// # Example + /// + /// A session compared with itself differs in nothing: + /// + /// ```rust + /// use komorebi_recorder::analysis::Analysis; + /// use komorebi_recorder::testkit::SessionBuilder; + /// + /// let (session, _) = SessionBuilder::new("doc-compare").length(5.0).build(); + /// let analysis = Analysis::open(&session.dir)?; + /// let data = analysis.compare(&analysis, None)?; + /// assert_eq!(data.a.duration_s, data.b.duration_s); + /// assert_eq!(data.deltas.duration_s, 0.0); + /// assert_eq!(data.deltas.findings_total, 0); + /// # Ok::<(), Box>(()) + /// ``` + pub fn compare( + &self, + other: &Analysis, + cancel: Option<&AtomicBool>, + ) -> eyre::Result { + let mut notes = Vec::new(); + let a = self.side(cancel, &mut notes)?; + let b = other.side(cancel, &mut notes)?; + let deltas = deltas(&a, &b); + Ok(CompareData { + a, + b, + deltas, + notes, + }) + } +} + +fn deltas(a: &Side, b: &Side) -> Deltas { + let both = |f: fn(&MotionMetrics) -> f64| match (&a.motion, &b.motion) { + (Some(x), Some(y)) => Some(f(y) - f(x)), + _ => None, + }; + let kinds: std::collections::BTreeSet<&String> = + a.findings.keys().chain(b.findings.keys()).collect(); + Deltas { + duration_s: b.duration_s - a.duration_s, + frames: b.frames as i64 - a.frames as i64, + effective_fps: b.effective_fps - a.effective_fps, + median_gap_ms: b.median_gap_ms - a.median_gap_ms, + worst_gap_ms: b.worst_gap_ms - a.worst_gap_ms, + late_gaps: b.late_gaps as i64 - a.late_gaps as i64, + motion_segments: both(|m| m.segments as f64).map(|d| d as i64), + active_s: both(|m| m.active_s), + peak_share: both(|m| m.peak_share), + peak_mean_delta: both(|m| m.peak_mean_delta), + findings_total: b.findings_total as i64 - a.findings_total as i64, + findings: kinds + .into_iter() + .map(|kind| { + let count = |side: &Side| side.findings.get(kind).copied().unwrap_or(0) as i64; + (kind.clone(), count(b) - count(a)) + }) + .collect(), + annotations: b.annotations as i64 - a.annotations as i64, + } +} + +impl CompareData { + /// Renders the comparison as a table with a column for each recording and one for the + /// difference. + /// + /// # Example + /// + /// ```rust + /// use komorebi_recorder::analysis::Analysis; + /// use komorebi_recorder::testkit::SessionBuilder; + /// + /// let (session, _) = SessionBuilder::new("doc-compare-text").length(5.0).build(); + /// let analysis = Analysis::open(&session.dir)?; + /// let text = analysis.compare(&analysis, None)?.to_text(); + /// assert!(text.contains("duration")); + /// assert!(text.contains("findings")); + /// # Ok::<(), Box>(()) + /// ``` + #[must_use] + pub fn to_text(&self) -> String { + use std::fmt::Write; + + let (a, b, d) = (&self.a, &self.b, &self.deltas); + let mut out = String::new(); + let _ = writeln!(out, "a: {}\nb: {}\n", a.session, b.session); + let _ = writeln!(out, "{:<24}{:>14}{:>14}{:>14}", "", "a", "b", "b - a"); + let mut row = |name: &str, x: String, y: String, delta: String| { + let _ = writeln!(out, "{name:<24}{x:>14}{y:>14}{delta:>14}"); + }; + let number = |value: f64, places: usize| format!("{value:.places$}"); + let signed = |value: f64, places: usize| format!("{value:+.places$}"); + let whole = |value: i64| format!("{value:+}"); + + row( + "duration (s)", + number(a.duration_s, 2), + number(b.duration_s, 2), + signed(d.duration_s, 2), + ); + row( + "frames", + a.frames.to_string(), + b.frames.to_string(), + whole(d.frames), + ); + row( + "effective fps", + number(a.effective_fps, 1), + number(b.effective_fps, 1), + signed(d.effective_fps, 1), + ); + row( + "median frame gap (ms)", + number(a.median_gap_ms, 1), + number(b.median_gap_ms, 1), + signed(d.median_gap_ms, 1), + ); + row( + "worst frame gap (ms)", + number(a.worst_gap_ms, 1), + number(b.worst_gap_ms, 1), + signed(d.worst_gap_ms, 1), + ); + row( + "late frame gaps", + a.late_gaps.to_string(), + b.late_gaps.to_string(), + whole(d.late_gaps), + ); + + let dash = || "-".to_string(); + row( + "motion segments", + a.motion + .as_ref() + .map_or_else(dash, |m| m.segments.to_string()), + b.motion + .as_ref() + .map_or_else(dash, |m| m.segments.to_string()), + d.motion_segments.map_or_else(dash, whole), + ); + row( + "time in motion (s)", + a.motion + .as_ref() + .map_or_else(dash, |m| number(m.active_s, 2)), + b.motion + .as_ref() + .map_or_else(dash, |m| number(m.active_s, 2)), + d.active_s.map_or_else(dash, |v| signed(v, 2)), + ); + row( + "peak motion (% pixels)", + a.motion + .as_ref() + .map_or_else(dash, |m| number(m.peak_share * 100.0, 1)), + b.motion + .as_ref() + .map_or_else(dash, |m| number(m.peak_share * 100.0, 1)), + d.peak_share.map_or_else(dash, |v| signed(v * 100.0, 1)), + ); + row( + "peak mean delta", + a.motion + .as_ref() + .map_or_else(dash, |m| number(m.peak_mean_delta, 1)), + b.motion + .as_ref() + .map_or_else(dash, |m| number(m.peak_mean_delta, 1)), + d.peak_mean_delta.map_or_else(dash, |v| signed(v, 1)), + ); + row( + "findings", + a.findings_total.to_string(), + b.findings_total.to_string(), + whole(d.findings_total), + ); + for (kind, delta) in &d.findings { + row( + &format!(" {kind}"), + a.findings.get(kind).copied().unwrap_or(0).to_string(), + b.findings.get(kind).copied().unwrap_or(0).to_string(), + whole(*delta), + ); + } + row( + "annotations", + a.annotations.to_string(), + b.annotations.to_string(), + whole(d.annotations), + ); + for note in &self.notes { + let _ = writeln!(out, "note: {note}"); + } + out + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn side(frames: usize, kinds: &[(&str, usize)]) -> Side { + Side { + session: String::new(), + duration_s: 10.0, + frames, + effective_fps: 30.0, + median_gap_ms: 33.0, + worst_gap_ms: 100.0, + worst_gap_at_video_s: None, + late_gaps: 0, + motion: None, + findings: kinds.iter().map(|(k, n)| ((*k).to_string(), *n)).collect(), + findings_total: kinds.iter().map(|(_, n)| n).sum(), + annotations: 0, + bugs: 0, + } + } + + #[test] + fn frame_gaps_ignore_headers_repeats_and_torn_rows() { + let (gaps, frames) = frame_gaps("0,100,100\n-,150,150\n1,300,300\n2,3"); + assert_eq!(frames, 2); + assert_eq!(gaps, [(100, 200)]); + } + + #[test] + fn deltas_cover_the_kinds_of_either_side() { + let a = side(100, &[("slow-handler", 2)]); + let b = side(80, &[("event-storm", 1), ("slow-handler", 1)]); + let d = deltas(&a, &b); + assert_eq!(d.frames, -20); + assert_eq!(d.findings_total, 0); + assert_eq!(d.findings["slow-handler"], -1); + assert_eq!(d.findings["event-storm"], 1); + assert_eq!(d.motion_segments, None); + } +} diff --git a/komorebi-recorder/src/analysis/framediff.rs b/komorebi-recorder/src/analysis/framediff.rs new file mode 100644 index 00000000..a1d9574a --- /dev/null +++ b/komorebi-recorder/src/analysis/framediff.rs @@ -0,0 +1,339 @@ +//! `framediff`: how two frames differ, in numbers and as a picture. +//! +//! Two frames are decoded (the ones on screen at two moments of one recording, or one moment of +//! each of two recordings), optionally cut to a region of the display, reduced to brightness and +//! compared pixel by pixel. The answer is how much changed (the share of pixels that moved by more +//! than 24 of 255, and the mean and largest brightness difference), the box around the change, and, +//! if asked, a PNG of the compared area in which the changed pixels are red. +//! +//! Frames are decoded at a width of 1280 pixels unless asked otherwise (never more than the +//! video has), so a region is compared at that resolution. Two recordings must have been made at +//! the same size to be compared. + +use super::Analysis; +use super::envelope; +use super::pixels; +use super::pixels::Crop; +use super::sheet::Canvas; +use super::video::SessionVideo; +use crate::av::Scale; +use color_eyre::eyre; +use schemars::JsonSchema; +use serde::Serialize; +use std::path::Path; + +/// The schema name that identifies a [`FrameDiffData`] in an envelope (`framediff/1`). +pub const SCHEMA_FRAMEDIFF: &str = "framediff/1"; + +/// The width frames are decoded at by default. +pub const DEFAULT_WIDTH: u32 = 1280; + +/// One frame to compare: a moment on a display of a recording. +#[derive(Clone, Copy)] +pub struct FrameAt<'a> { + /// The recording. + pub analysis: &'a Analysis, + /// Its video, made on this thread. + pub video: &'a SessionVideo<'a>, + /// The display. + pub display: u32, + /// The moment, in seconds of video time of that recording. + pub time_s: f64, +} + +/// What to compare and what to write. +#[derive(Debug, Clone, Copy)] +pub struct FrameDiffOptions<'a> { + /// Only this part of the display, in global display points (of the first frame's display); + /// the whole display if `None`. + pub crop: Option, + /// The width to decode frames at. + pub width: u32, + /// Where to write the PNG of the change, if wanted. + pub png: Option<&'a Path>, +} + +/// How two frames differ: the `data` of `framediff`. +#[derive(Debug, Clone, PartialEq, Serialize, JsonSchema)] +pub struct FrameDiffData { + /// The display of the first frame. + pub display: u32, + /// The display of the second frame (differs only when `against` was used). + pub display_b: u32, + /// The moment asked for the first frame, in seconds of video time. + pub time_a_s: f64, + /// When the frame that was on screen then was shown. + pub shown_a_s: f64, + /// The moment asked for the second frame, in seconds of video time. + pub time_b_s: f64, + /// When the frame that was on screen then was shown. + pub shown_b_s: f64, + /// The other recording, if the second frame is from one. + pub against: Option, + /// The region compared, in global display points; `None` is the whole display. + pub region: Option, + /// The width of the compared area, in pixels. + pub width: u32, + /// The height of the compared area, in pixels. + pub height: u32, + /// How many pixels moved by more than 24 of 255 in brightness. + pub changed_pixels: usize, + /// The share of the compared pixels that changed, 0 to 1. + pub changed_share: f64, + /// The mean absolute brightness difference over the compared pixels, 0 to 255. + pub mean_delta: f64, + /// The largest brightness difference of any pixel, 0 to 255. + pub peak_delta: u8, + /// The box around the changed pixels as fractions of the display: `[x, y, width, height]`. + /// `None` if nothing changed. + pub bbox: Option<[f64; 4]>, + /// The same box in global display points. + pub bbox_points: Option<[f64; 4]>, + /// Where the PNG was written, if one was asked for. + pub png: Option, +} + +impl FrameDiffData { + /// Renders the answer as text. + /// + /// # Example + /// + /// ```rust + /// use komorebi_recorder::analysis::framediff::FrameDiffData; + /// + /// let data = FrameDiffData { + /// display: 1, display_b: 1, time_a_s: 3.2, shown_a_s: 3.18, time_b_s: 3.5, shown_b_s: 3.47, + /// against: None, region: None, width: 1280, height: 540, changed_pixels: 6912, + /// changed_share: 0.01, mean_delta: 2.5, peak_delta: 200, bbox: Some([0.1, 0.1, 0.5, 0.5]), + /// bbox_points: None, png: None, + /// }; + /// assert!(data.to_text().contains("1.00% of the pixels changed")); + /// ``` + #[must_use] + pub fn to_text(&self) -> String { + use crate::timecode::format as time; + use std::fmt::Write; + + let mut out = String::new(); + let _ = writeln!( + out, + "display {}: {} (frame shown {}) against {} (frame shown {}){}", + self.display, + time(self.time_a_s), + time(self.shown_a_s), + time(self.time_b_s), + time(self.shown_b_s), + self.against + .as_ref() + .map_or_else(String::new, |other| format!(" of {other}")), + ); + let _ = writeln!( + out, + "{:.2}% of the pixels changed ({} of {}x{}), mean brightness delta {:.2}, peak {}", + self.changed_share * 100.0, + self.changed_pixels, + self.width, + self.height, + self.mean_delta, + self.peak_delta, + ); + match (self.bbox, self.bbox_points) { + (Some(bbox), points) => { + let _ = write!( + out, + "box of change {:.0}%,{:.0}% {:.0}%x{:.0}% of the display", + bbox[0] * 100.0, + bbox[1] * 100.0, + bbox[2] * 100.0, + bbox[3] * 100.0, + ); + if let Some(p) = points { + let _ = write!( + out, + " ({:.0},{:.0} {:.0}x{:.0} points)", + p[0], p[1], p[2], p[3] + ); + } + out.push('\n'); + } + (None, _) => out.push_str("nothing changed\n"), + } + if let Some(path) = &self.png { + let _ = writeln!(out, "wrote {path}"); + } + out + } +} + +fn grab(at: &FrameAt, scale: Scale, what: &str) -> eyre::Result { + at.video + .frame_at(at.display, at.time_s, scale)? + .ok_or_else(|| { + envelope::not_found(format!( + "display {} had no video at {} ({what})", + at.display, + crate::timecode::format(at.time_s) + )) + }) +} + +/// Compares the frame `a` with the frame `b`, as the [module](self) describes. +/// +/// The region is placed on the display of `a` (so it is in the points of that recording) and +/// applied as fractions to both. +/// +/// # Errors +/// +/// Returns an error if: +/// +/// - the region does not overlap the display, or its frame is unknown (the usage kind); +/// - a display had no video at its moment (not found); +/// - the two frames are not the same size, which happens when the recordings were made at +/// different sizes (the unusable kind); +/// - a video cannot be opened or decoded, the host has no decoder, or the PNG cannot be written. +/// +/// # Example +/// +/// This needs recorded video, so it is not run here; `tests/video.rs` runs it through the +/// `framediff` command. +/// +/// ```rust,no_run +/// use komorebi_recorder::analysis::Analysis; +/// use komorebi_recorder::analysis::framediff::{DEFAULT_WIDTH, FrameAt, FrameDiffOptions, framediff}; +/// use komorebi_recorder::analysis::video::SessionVideo; +/// +/// let analysis = Analysis::open(std::path::Path::new("session-dir"))?; +/// let video = SessionVideo::new(&analysis.manifest, analysis.guard()); +/// let at = |time_s| FrameAt { analysis: &analysis, video: &video, display: 1, time_s }; +/// let data = framediff( +/// &at(3.2), +/// &at(3.5), +/// &FrameDiffOptions { crop: None, width: DEFAULT_WIDTH, png: None }, +/// )?; +/// print!("{}", data.to_text()); +/// # Ok::<(), Box>(()) +/// ``` +pub fn framediff( + a: &FrameAt, + b: &FrameAt, + options: &FrameDiffOptions, +) -> eyre::Result { + let fractions = options + .crop + .as_ref() + .map(|crop| a.analysis.crop_fractions(a.display, crop)) + .transpose()?; + let scale = Scale::Width(options.width.max(2)); + let (first, second) = (grab(a, scale, "first")?, grab(b, scale, "second")?); + let (width, height) = (first.frame.width, first.frame.height); + if (second.frame.width, second.frame.height) != (width, height) { + return Err(envelope::unusable(format!( + "the frames are not the same size ({width}x{height} against {}x{}): the recordings were made at different sizes", + second.frame.width, second.frame.height + ))); + } + + let rect = fractions.map_or( + pixels::PixelRect { + x: 0, + y: 0, + width, + height, + }, + |fractions| pixels::pixel_rect(fractions, width, height), + ); + let before = pixels::crop_luma(&pixels::luma(&first.frame.bgra), width, rect); + let after = pixels::crop_luma(&pixels::luma(&second.frame.bgra), width, rect); + let found = pixels::delta(&before, &after, rect.width, super::changes::PIXEL_THRESHOLD); + + let bbox = found.bounds.map(|(min_x, min_y, max_x, max_y)| { + super::changes::fractions_of( + ( + min_x + rect.x, + min_y + rect.y, + max_x + rect.x, + max_y + rect.y, + ), + Some((width, height)), + ) + }); + + if let Some(path) = options.png { + write_png( + path, + &second.frame.bgra, + width, + rect, + &before, + &after, + found.bounds, + )?; + } + + Ok(FrameDiffData { + display: a.display, + display_b: b.display, + time_a_s: a.time_s, + shown_a_s: first.shown_s, + time_b_s: b.time_s, + shown_b_s: second.shown_s, + against: (!std::ptr::eq(a.analysis, b.analysis)) + .then(|| b.analysis.session.dir.display().to_string()), + region: options.crop, + width: rect.width, + height: rect.height, + changed_pixels: found.changed, + changed_share: found.share(), + mean_delta: found.mean, + peak_delta: found.peak, + bbox, + bbox_points: bbox + .and_then(|bbox| super::changes::points_of(a.analysis.display_frame(a.display), bbox)), + png: options.png.map(|path| path.display().to_string()), + }) +} + +/// Writes the compared area of the second frame, dimmed, with the pixels that changed in red and +/// a yellow box around them. +fn write_png( + path: &Path, + bgra: &[u8], + width: u32, + rect: pixels::PixelRect, + before: &[u8], + after: &[u8], + bounds: Option<(u32, u32, u32, u32)>, +) -> eyre::Result<()> { + let area = pixels::crop_bgra(bgra, width, rect); + let mut canvas = Canvas::new(rect.width, rect.height); + for row in 0..rect.height { + for column in 0..rect.width { + let at = (row * rect.width + column) as usize; + let changed = (i32::from(before[at]) - i32::from(after[at])).abs() + > super::changes::PIXEL_THRESHOLD; + let color = if changed { + [255, 40, 40] + } else { + let grey = (u32::from(area[at * 4 + 2]) * 299 + + u32::from(area[at * 4 + 1]) * 587 + + u32::from(area[at * 4]) * 114) + / 1000; + let dimmed = (grey * 2 / 5) as u8; + [dimmed, dimmed, dimmed] + }; + canvas.put(i64::from(column), i64::from(row), color, 1.0); + } + } + if let Some((min_x, min_y, max_x, max_y)) = bounds { + canvas.border( + min_x, + min_y, + max_x - min_x + 1, + max_y - min_y + 1, + 1, + [255, 220, 0], + ); + } + + canvas.save(path) +} diff --git a/komorebi-recorder/src/analysis/mod.rs b/komorebi-recorder/src/analysis/mod.rs index 8c6b94f9..cd98e6fa 100644 --- a/komorebi-recorder/src/analysis/mod.rs +++ b/komorebi-recorder/src/analysis/mod.rs @@ -8,6 +8,7 @@ //! bound ([`budget`]). pub mod answers; +pub mod blank; pub mod blocks; pub mod brief; pub mod budget; @@ -15,11 +16,13 @@ pub mod catalog; pub mod changes; pub mod classify; pub mod code; +pub mod compare; pub mod diff; pub mod envelope; pub mod events; pub mod facade; pub mod flavor; +pub mod framediff; pub mod ghost; pub mod guard; pub mod history; @@ -28,8 +31,10 @@ pub mod invariants; pub mod logparse; pub mod logq; pub mod lookup; +pub mod motion; pub mod os_trace; pub mod overview; +pub mod pixels; pub mod probed; pub mod schemas; pub mod search; diff --git a/komorebi-recorder/src/analysis/motion.rs b/komorebi-recorder/src/analysis/motion.rs new file mode 100644 index 00000000..1f11a3c1 --- /dev/null +++ b/komorebi-recorder/src/analysis/motion.rs @@ -0,0 +1,417 @@ +//! `motion`: when the picture of a display, or of a part of it, was moving. +//! +//! This is [`changes`](super::changes) with a region: the same frame-to-frame comparison, but it +//! can be limited to a rectangle of a display ([`ScanOptions::crop`], in display points), and each +//! segment of activity reports how hard the picture moved as well as how much of it did. +//! +//! A segment is a run of frames in which at least 0.05% (divided by the sensitivity) of the +//! compared pixels changed by more than 24 of 255 brightness, with frames closer than 0.15 s +//! belonging to one segment; see [`super::changes`] for the details. Per segment it reports: +//! +//! - `peak_share`: the largest share of the compared pixels that changed between two frames; +//! - `peak_mean_delta`: the largest mean brightness difference (0 to 255) between two frames, +//! which tells a faint flicker over the whole region from a few pixels flipping fully. +//! +//! Where a region is given the compared pixels are those of the region only, so `peak_share` is a +//! share of the region. The box of change is still given as a fraction of the display and in +//! display points, so it can be used with `--region` of the other commands. +//! +//! A region is cut from a frame decoded wide enough to give it at least 240 pixels across, so a +//! small region costs more to scan than a whole display. + +use super::Analysis; +use super::changes::Gap; +use super::changes::Tracking; +use super::changes::fractions_of; +use super::changes::points_of; +use super::changes::track; +use super::envelope; +use super::pixels::Crop; +use super::video::SessionVideo; +use crate::av::Scale; +use color_eyre::eyre; +use schemars::JsonSchema; +use serde::Serialize; +use std::sync::atomic::AtomicBool; + +/// The schema name that identifies a [`MotionData`] in an envelope (`motion/1`). +pub const SCHEMA_MOTION: &str = "motion/1"; + +/// Whole displays are compared at this width, like `changes`. +const WHOLE_WIDTH: u32 = 240; + +/// A region is decoded wide enough to have this many pixels across. +const REGION_PIXELS: f64 = 240.0; + +/// The widest a frame is decoded for a region. +const MAX_WIDTH: u32 = 3840; + +/// What to scan: shared by `motion` and `blank`. +/// +/// `crop` is where a window's rect goes, once a window track can say where a window was: it is in +/// the same global display points as a `bbox_points`, so a [`Crop::from_rect`] of a window's rect +/// is all it takes. +#[derive(Debug, Clone, Copy)] +pub struct ScanOptions<'a> { + /// The displays to scan; each must be a display of the session. + pub displays: &'a [u32], + /// The start of the range, in seconds of video time. + pub from_s: f64, + /// The end of the range, in seconds of video time. + pub to_s: f64, + /// Only this part of each display, in global display points; the whole display if `None`. + pub crop: Option, + /// Set from another thread to stop early with what was found so far. + pub cancel: Option<&'a AtomicBool>, +} + +/// A stretch of video in which the picture of a display, or of a region of it, was moving. +#[derive(Debug, Clone, PartialEq, Serialize, JsonSchema)] +pub struct MotionSegment { + /// The id of the display. + pub display: u32, + /// When the first changed frame was on screen, in seconds of video time. + pub start_video_s: f64, + /// When the last changed frame was on screen, in seconds of video time. + pub end_video_s: f64, + /// `end_video_s - start_video_s`; zero for a single changed frame. + pub duration_s: f64, + /// Changed frames in the segment. + pub frames: usize, + /// The largest share of the compared pixels (the region's, if there is one) that changed + /// between two consecutive frames. + pub peak_share: f64, + /// The largest mean brightness difference, 0 to 255, over the compared pixels between two + /// consecutive frames. + pub peak_mean_delta: f64, + /// The box around everything that changed, as fractions of the display: `[x, y, width, height]`. + pub bbox: [f64; 4], + /// The same box in global display points, if the display's frame is known. + pub bbox_points: Option<[f64; 4]>, + /// The `seq` of each komorebi event received within 0.25 s of the segment. + pub events: Vec, +} + +/// Where a display, or a region of it, was moving over a range: the `data` of `motion`. +#[derive(Debug, Clone, PartialEq, Serialize, JsonSchema)] +pub struct MotionData { + /// The start of the range scanned, in seconds of video time. + pub from_video_s: f64, + /// The end of the range scanned, in seconds of video time. + pub to_video_s: f64, + /// The displays scanned. + pub displays: Vec, + /// The region compared, in global display points, if the scan was limited to one. + pub region: Option, + /// The sensitivity asked for. + pub sensitivity: f64, + /// How many frames were decoded and looked at, over all displays. A cancelled scan counts + /// only what it got through. + pub frames_examined: usize, + /// The segments of activity, ordered by start and then display. + pub segments: Vec, + /// Stretches of the range where a display had no video at all. + pub gaps: Vec, +} + +impl MotionData { + /// The total time covered by segments, in seconds, counting each display's segments (they + /// never overlap within a display) and adding displays together. + #[must_use] + pub fn active_s(&self) -> f64 { + self.segments.iter().map(|segment| segment.duration_s).sum() + } + + /// Renders the answer as text, one line for each segment. + /// + /// # Example + /// + /// ```rust + /// use komorebi_recorder::analysis::motion::{MotionData, MotionSegment}; + /// + /// let data = MotionData { + /// from_video_s: 0.0, + /// to_video_s: 6.0, + /// displays: vec![1], + /// region: None, + /// sensitivity: 1.0, + /// frames_examined: 100, + /// segments: vec![MotionSegment { + /// display: 1, + /// start_video_s: 1.0, + /// end_video_s: 1.5, + /// duration_s: 0.5, + /// frames: 10, + /// peak_share: 0.25, + /// peak_mean_delta: 12.5, + /// bbox: [0.0, 0.0, 1.0, 1.0], + /// bbox_points: None, + /// events: vec![], + /// }], + /// gaps: vec![], + /// }; + /// let text = data.to_text(); + /// assert!(text.starts_with("100 frames compared on display(s) 1")); + /// assert!(text.contains("peak 25.0% of the pixels, mean delta 12.5")); + /// ``` + #[must_use] + pub fn to_text(&self) -> String { + use crate::timecode::format as time; + use std::fmt::Write; + + let mut out = String::new(); + let region = self.region.map_or_else(String::new, |crop| { + format!( + " in region {},{} {}x{}", + crop.x, crop.y, crop.width, crop.height + ) + }); + let _ = writeln!( + out, + "{} frames compared on display(s) {}{region} from {} to {}: {} segment(s) of motion, {:.2} s in all", + self.frames_examined, + self.displays + .iter() + .map(ToString::to_string) + .collect::>() + .join(", "), + time(self.from_video_s), + time(self.to_video_s), + self.segments.len(), + self.active_s(), + ); + for segment in &self.segments { + let events = if segment.events.is_empty() { + String::new() + } else { + format!( + " events {}", + segment + .events + .iter() + .map(|seq| format!("#{seq}")) + .collect::>() + .join(" ") + ) + }; + let _ = writeln!( + out, + "display {} {} - {} ({:.2} s, {} frames) peak {:.1}% of the pixels, mean delta {:.1} box {:.0}%,{:.0}% {:.0}%x{:.0}%{events}", + segment.display, + time(segment.start_video_s), + time(segment.end_video_s), + segment.duration_s, + segment.frames, + segment.peak_share * 100.0, + segment.peak_mean_delta, + segment.bbox[0] * 100.0, + segment.bbox[1] * 100.0, + segment.bbox[2] * 100.0, + segment.bbox[3] * 100.0, + ); + } + for gap in &self.gaps { + let _ = writeln!( + out, + "display {} has no video from {} to {}", + gap.display, + time(gap.from_video_s), + time(gap.to_video_s) + ); + } + out + } +} + +/// The width to decode a frame at so that a region of fractional width `fraction` is at least +/// [`REGION_PIXELS`] across. +pub(super) fn scale_for(fraction: f64) -> Scale { + let wanted = (REGION_PIXELS / fraction.max(1e-6)).ceil(); + Scale::Width((wanted as u32).clamp(WHOLE_WIDTH, MAX_WIDTH)) +} + +impl Analysis { + /// The frame of a display in global points, `[x, y, width, height]`, if the session knows it. + pub(super) fn display_frame(&self, display: u32) -> Option<[f64; 4]> { + self.manifest + .displays + .iter() + .find(|entry| entry.result.display.id == display) + .map(|entry| entry.result.display.frame) + } + + /// Turns a crop in points into the fractions of `display`. + pub(super) fn crop_fractions(&self, display: u32, crop: &Crop) -> eyre::Result<[f64; 4]> { + let frame = self.display_frame(display).ok_or_else(|| { + envelope::usage(format!( + "display {display} has no known frame, so a region in points cannot be placed on it" + )) + })?; + crop.fractions(frame) + } + + /// Finds the segments of activity on each display of `options`, in the whole display or in + /// `options.crop`, with `sensitivity` as for [`Analysis::changes`]. + /// + /// Stops early, without an error, once `options.cancel` is set; [`MotionData::frames_examined`] + /// says how far it got. A display with no video in the range contributes no segments and is + /// reported in [`MotionData::gaps`]. + /// + /// # Errors + /// + /// Returns an error if: + /// + /// - `sensitivity` is not above zero (the usage exit kind); + /// - the crop does not overlap a display, or a display's frame is unknown (the usage kind); + /// - a video file cannot be resolved inside the session, opened or decoded, or the host has no + /// decoder. + /// + /// # Example + /// + /// This needs recorded video, so it is not run here. `tests/video.rs` runs it through the + /// `motion` command on sessions with numbered videos. + /// + /// ```rust,no_run + /// use komorebi_recorder::analysis::Analysis; + /// use komorebi_recorder::analysis::motion::ScanOptions; + /// use komorebi_recorder::analysis::pixels::Crop; + /// use komorebi_recorder::analysis::video::SessionVideo; + /// + /// let analysis = Analysis::open(std::path::Path::new("session-dir"))?; + /// let video = SessionVideo::new(&analysis.manifest, analysis.guard()); + /// let data = analysis.motion( + /// &video, + /// &ScanOptions { + /// displays: &video.displays(), + /// from_s: 0.0, + /// to_s: 60.0, + /// crop: Some(Crop::parse("100,100,800,600")?), + /// cancel: None, + /// }, + /// 1.0, + /// )?; + /// print!("{}", data.to_text()); + /// # Ok::<(), Box>(()) + /// ``` + pub fn motion( + &self, + video: &SessionVideo, + options: &ScanOptions, + sensitivity: f64, + ) -> eyre::Result { + if sensitivity <= 0.0 { + return Err(envelope::usage("--sensitivity must be above zero")); + } + + let threshold = super::changes::SHARE_THRESHOLD / sensitivity; + let mut segments = Vec::new(); + let mut examined = 0; + for display in options.displays { + let (crop, scale) = match &options.crop { + Some(crop) => { + let fractions = self.crop_fractions(*display, crop)?; + (Some(fractions), scale_for(fractions[2])) + } + None => (None, Scale::Width(WHOLE_WIDTH)), + }; + let tracked = track( + video, + *display, + options.from_s, + options.to_s, + &Tracking { + threshold, + crop, + scale, + cancel: options.cancel, + }, + )?; + examined += tracked.examined; + + let frame_box = self.display_frame(*display); + for span in tracked.spans { + let bbox = fractions_of(span.bounds, tracked.dimensions); + segments.push(MotionSegment { + display: *display, + start_video_s: span.start, + end_video_s: span.end, + duration_s: span.end - span.start, + frames: span.frames, + peak_share: span.peak, + peak_mean_delta: span.peak_delta, + bbox, + bbox_points: points_of(frame_box, bbox), + events: self.events_between(span.start, span.end), + }); + } + } + + segments.sort_by(|a, b| { + a.start_video_s + .total_cmp(&b.start_video_s) + .then(a.display.cmp(&b.display)) + }); + + Ok(MotionData { + from_video_s: options.from_s, + to_video_s: options.to_s, + displays: options.displays.to_vec(), + region: options.crop, + sensitivity, + frames_examined: examined, + segments, + gaps: self.video_gaps(options.displays, options.from_s, options.to_s), + }) + } + + /// The `seq` of each event received within a quarter of a second of `start_s..=end_s`. + fn events_between(&self, start_s: f64, end_s: f64) -> Vec { + self.events() + .iter() + .filter(|event| { + let at = self.video_s(event.recv_mach_ns); + at >= start_s - 0.25 && at <= end_s + 0.25 + }) + .map(|event| event.seq) + .collect() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_narrow_region_is_decoded_wider_but_never_past_the_limit() { + assert_eq!(scale_for(1.0), Scale::Width(240)); + assert_eq!(scale_for(0.5), Scale::Width(480)); + assert_eq!(scale_for(0.0001), Scale::Width(MAX_WIDTH)); + } + + #[test] + fn active_time_adds_up_the_segments() { + let segment = |start: f64, end: f64| MotionSegment { + display: 1, + start_video_s: start, + end_video_s: end, + duration_s: end - start, + frames: 1, + peak_share: 0.0, + peak_mean_delta: 0.0, + bbox: [0.0; 4], + bbox_points: None, + events: vec![], + }; + let data = MotionData { + from_video_s: 0.0, + to_video_s: 10.0, + displays: vec![1], + region: None, + sensitivity: 1.0, + frames_examined: 0, + segments: vec![segment(1.0, 1.5), segment(3.0, 4.0)], + gaps: vec![], + }; + assert_eq!(data.active_s(), 1.5); + } +} diff --git a/komorebi-recorder/src/analysis/pixels.rs b/komorebi-recorder/src/analysis/pixels.rs new file mode 100644 index 00000000..99a20cf3 --- /dev/null +++ b/komorebi-recorder/src/analysis/pixels.rs @@ -0,0 +1,438 @@ +//! Pixel arithmetic shared by the commands that look at frames: brightness, cropping to a region +//! of a display, and comparing two pictures. +//! +//! Nothing here decodes video, so it can be tested on synthetic frames. The commands that use it +//! are `changes`, `motion`, `framediff` and `blank` ([`super::changes`], [`super::motion`], +//! [`super::framediff`], [`super::blank`]). +//! +//! # Regions +//! +//! A [`Crop`] is a rectangle in *display points*: the same global coordinates as +//! [`ChangeSpan::bbox_points`](super::changes::ChangeSpan::bbox_points) and a window's rect in +//! komorebi's state. It is turned into fractions of a display's frame ([`Crop::fractions`]) and +//! then into pixels of whatever size a frame was decoded at ([`pixel_rect`]), so the same crop +//! works at any decode size. A caller that has a window's rect passes it to [`Crop::from_rect`]. +//! +//! # Example +//! +//! ```rust +//! use komorebi_recorder::analysis::pixels::{Crop, pixel_rect}; +//! +//! // the right half of a display whose frame is 1000 x 500 points at (100, 0) +//! let crop = Crop::parse("600,0,500,500")?; +//! let fractions = crop.fractions([100.0, 0.0, 1000.0, 500.0])?; +//! assert_eq!(fractions, [0.5, 0.0, 0.5, 1.0]); +//! +//! // in a frame decoded at 240 x 120 pixels +//! let rect = pixel_rect(fractions, 240, 120); +//! assert_eq!((rect.x, rect.y, rect.width, rect.height), (120, 0, 120, 120)); +//! # Ok::<(), Box>(()) +//! ``` + +use super::envelope; +use color_eyre::eyre; +use schemars::JsonSchema; +use serde::Serialize; + +/// A rectangle in global display points: `x`, `y`, `width`, `height`. +#[derive(Debug, Clone, Copy, PartialEq, Serialize, JsonSchema)] +pub struct Crop { + /// Left edge, in points. + pub x: f64, + /// Top edge, in points. + pub y: f64, + /// Width, in points; above zero. + pub width: f64, + /// Height, in points; above zero. + pub height: f64, +} + +impl Crop { + /// Returns the crop for a rect of `[x, y, width, height]`, such as a window's rect. + #[must_use] + pub fn from_rect(rect: [f64; 4]) -> Self { + Self { + x: rect[0], + y: rect[1], + width: rect[2], + height: rect[3], + } + } + + /// Returns the rect as `[x, y, width, height]`. + #[must_use] + pub fn rect(&self) -> [f64; 4] { + [self.x, self.y, self.width, self.height] + } + + /// Parses `x,y,w,h` (numbers, in points). + /// + /// # Errors + /// + /// Returns a usage error if the text is not four numbers or the size is not above zero. + /// + /// # Example + /// + /// ```rust + /// use komorebi_recorder::analysis::pixels::Crop; + /// + /// assert_eq!(Crop::parse("10, 20,300,40")?.rect(), [10.0, 20.0, 300.0, 40.0]); + /// assert!(Crop::parse("1,2,3").is_err()); + /// assert!(Crop::parse("1,2,0,4").is_err()); + /// # Ok::<(), Box>(()) + /// ``` + pub fn parse(text: &str) -> eyre::Result { + let numbers: Vec = text + .split(',') + .map(|part| part.trim().parse::()) + .collect::>() + .map_err(|_| { + envelope::usage(format!("--region wants x,y,w,h in points, not {text:?}")) + })?; + let [x, y, width, height] = numbers[..] else { + return Err(envelope::usage(format!( + "--region wants four numbers, x,y,w,h, not {text:?}" + ))); + }; + if !(width > 0.0 && height > 0.0 && x.is_finite() && y.is_finite()) { + return Err(envelope::usage( + "--region needs a width and height above zero", + )); + } + + Ok(Self { + x, + y, + width, + height, + }) + } + + /// Returns the crop as fractions of a display whose frame is `frame` (`[x, y, width, height]` + /// in points): `[x, y, width, height]`, each from 0 to 1. The part of the crop outside the + /// display is cut off. + /// + /// # Errors + /// + /// Returns a usage error if the frame has no size or the crop does not overlap the display. + pub fn fractions(&self, frame: [f64; 4]) -> eyre::Result<[f64; 4]> { + if frame[2] <= 0.0 || frame[3] <= 0.0 { + return Err(envelope::usage("the display's frame has no size")); + } + + let left = ((self.x - frame[0]) / frame[2]).clamp(0.0, 1.0); + let top = ((self.y - frame[1]) / frame[3]).clamp(0.0, 1.0); + let right = ((self.x + self.width - frame[0]) / frame[2]).clamp(0.0, 1.0); + let bottom = ((self.y + self.height - frame[1]) / frame[3]).clamp(0.0, 1.0); + if right <= left || bottom <= top { + return Err(envelope::usage(format!( + "the region {},{},{},{} is outside the display ({},{} {}x{} points)", + self.x, self.y, self.width, self.height, frame[0], frame[1], frame[2], frame[3] + ))); + } + + Ok([left, top, right - left, bottom - top]) + } +} + +/// A rectangle of pixels in a decoded frame. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct PixelRect { + /// Left edge. + pub x: u32, + /// Top edge. + pub y: u32, + /// Width, at least 1. + pub width: u32, + /// Height, at least 1. + pub height: u32, +} + +/// Converts fractions of a frame to the pixels of a `width` x `height` frame. The rect is at +/// least one pixel and never leaves the frame. +#[must_use] +pub fn pixel_rect(fractions: [f64; 4], width: u32, height: u32) -> PixelRect { + let (w, h) = (width.max(1), height.max(1)); + let x = ((fractions[0] * f64::from(w)).floor() as u32).min(w - 1); + let y = ((fractions[1] * f64::from(h)).floor() as u32).min(h - 1); + let right = (((fractions[0] + fractions[2]) * f64::from(w)).ceil() as u32).clamp(x + 1, w); + let bottom = (((fractions[1] + fractions[3]) * f64::from(h)).ceil() as u32).clamp(y + 1, h); + PixelRect { + x, + y, + width: right - x, + height: bottom - y, + } +} + +/// The brightness (Rec. 601 weights, integers) of every pixel of a BGRA picture. +/// +/// # Example +/// +/// ```rust +/// use komorebi_recorder::analysis::pixels::luma; +/// +/// // one white pixel, one black, one pure green (BGRA order) +/// assert_eq!(luma(&[255, 255, 255, 255, 0, 0, 0, 255, 0, 255, 0, 255]), [255, 0, 149]); +/// ``` +#[must_use] +pub fn luma(bgra: &[u8]) -> Vec { + bgra.as_chunks::<4>() + .0 + .iter() + .map(|pixel| { + ((u32::from(pixel[2]) * 299 + u32::from(pixel[1]) * 587 + u32::from(pixel[0]) * 114) + / 1000) as u8 + }) + .collect() +} + +/// The brightness of `rect` of a picture whose rows are `width` pixels, row by row. +#[must_use] +pub fn crop_luma(luma: &[u8], width: u32, rect: PixelRect) -> Vec { + let mut out = Vec::with_capacity(rect.width as usize * rect.height as usize); + for row in rect.y..rect.y + rect.height { + let start = row as usize * width as usize + rect.x as usize; + if let Some(line) = luma.get(start..start + rect.width as usize) { + out.extend_from_slice(line); + } + } + out +} + +/// The BGRA bytes of `rect` of a BGRA picture whose rows are `width` pixels. +#[must_use] +pub fn crop_bgra(bgra: &[u8], width: u32, rect: PixelRect) -> Vec { + let mut out = Vec::with_capacity(rect.width as usize * rect.height as usize * 4); + for row in rect.y..rect.y + rect.height { + let start = (row as usize * width as usize + rect.x as usize) * 4; + if let Some(line) = bgra.get(start..start + rect.width as usize * 4) { + out.extend_from_slice(line); + } + } + out +} + +/// How two same-sized brightness pictures differ. +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct Delta { + /// How many pixels moved by more than the threshold. + pub changed: usize, + /// How many pixels were compared. + pub total: usize, + /// The mean of the absolute brightness difference over all pixels, 0 to 255. + pub mean: f64, + /// The largest absolute brightness difference of any pixel. + pub peak: u8, + /// The box around the changed pixels, `(min_x, min_y, max_x, max_y)` in pixels of the + /// pictures, or `None` if nothing changed. + pub bounds: Option<(u32, u32, u32, u32)>, +} + +impl Delta { + /// The share of pixels that changed, 0 to 1. + #[must_use] + pub fn share(&self) -> f64 { + self.changed as f64 / self.total.max(1) as f64 + } +} + +/// Compares two brightness pictures `width` pixels wide. A pixel has changed if its brightness +/// moved by more than `threshold`. Pictures of different lengths are compared as far as the +/// shorter goes. +/// +/// # Example +/// +/// ```rust +/// use komorebi_recorder::analysis::pixels::delta; +/// +/// // a 2 x 2 picture where the bottom right pixel went from black to white +/// let found = delta(&[0, 0, 0, 0], &[0, 0, 0, 255], 2, 24); +/// assert_eq!(found.changed, 1); +/// assert_eq!(found.share(), 0.25); +/// assert_eq!(found.peak, 255); +/// assert_eq!(found.bounds, Some((1, 1, 1, 1))); +/// ``` +#[must_use] +pub fn delta(before: &[u8], after: &[u8], width: u32, threshold: i32) -> Delta { + let (mut changed, mut sum, mut peak) = (0_usize, 0_u64, 0_u8); + let (mut min_x, mut min_y, mut max_x, mut max_y) = (u32::MAX, u32::MAX, 0_u32, 0_u32); + let width = width.max(1); + for (index, (a, b)) in before.iter().zip(after).enumerate() { + let difference = (i32::from(*a) - i32::from(*b)).unsigned_abs() as u8; + sum += u64::from(difference); + peak = peak.max(difference); + if i32::from(difference) > threshold { + let (x, y) = (index as u32 % width, index as u32 / width); + changed += 1; + min_x = min_x.min(x); + max_x = max_x.max(x); + min_y = min_y.min(y); + max_y = max_y.max(y); + } + } + + let total = before.len().min(after.len()); + Delta { + changed, + total, + mean: sum as f64 / total.max(1) as f64, + peak, + bounds: (changed > 0).then_some((min_x, min_y, max_x, max_y)), + } +} + +/// How even a patch of the screen is. +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct Evenness { + /// The variance of brightness over the patch. + pub variance: f64, + /// The mean colour, `[red, green, blue]`. + pub rgb: [u8; 3], +} + +/// Measures a BGRA patch: the variance of its brightness and its mean colour. +/// +/// # Example +/// +/// ```rust +/// use komorebi_recorder::analysis::pixels::evenness; +/// +/// let white = [255_u8; 16]; // four white pixels +/// let found = evenness(&white); +/// assert_eq!(found.variance, 0.0); +/// assert_eq!(found.rgb, [255, 255, 255]); +/// ``` +#[must_use] +pub fn evenness(bgra: &[u8]) -> Evenness { + let pixels = bgra.as_chunks::<4>().0; + let count = pixels.len().max(1) as f64; + let (mut r, mut g, mut b) = (0_u64, 0_u64, 0_u64); + let (mut sum, mut square) = (0_f64, 0_f64); + for pixel in pixels { + r += u64::from(pixel[2]); + g += u64::from(pixel[1]); + b += u64::from(pixel[0]); + let l = f64::from( + (u32::from(pixel[2]) * 299 + u32::from(pixel[1]) * 587 + u32::from(pixel[0]) * 114) + / 1000, + ); + sum += l; + square += l * l; + } + + let mean = sum / count; + Evenness { + variance: (square / count - mean * mean).max(0.0), + rgb: [ + (r as f64 / count).round() as u8, + (g as f64 / count).round() as u8, + (b as f64 / count).round() as u8, + ], + } +} + +/// Names a colour a person would: `white`, `black` or `gray` when the channels agree, otherwise +/// the hex code (`#1a2b3c`). +/// +/// # Example +/// +/// ```rust +/// use komorebi_recorder::analysis::pixels::colour_name; +/// +/// assert_eq!(colour_name([250, 252, 255]), "white"); +/// assert_eq!(colour_name([3, 3, 3]), "black"); +/// assert_eq!(colour_name([128, 128, 128]), "gray"); +/// assert_eq!(colour_name([200, 0, 16]), "#c80010"); +/// ``` +#[must_use] +pub fn colour_name(rgb: [u8; 3]) -> String { + let (low, high) = ( + rgb.iter().copied().min().unwrap_or(0), + rgb.iter().copied().max().unwrap_or(0), + ); + if high - low <= 12 { + let mean = (u32::from(rgb[0]) + u32::from(rgb[1]) + u32::from(rgb[2])) / 3; + return match mean { + 224.. => "white", + ..32 => "black", + _ => "gray", + } + .to_string(); + } + + format!("#{:02x}{:02x}{:02x}", rgb[0], rgb[1], rgb[2]) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_crop_is_cut_off_at_the_display_and_refused_outside_it() { + let frame = [0.0, 0.0, 100.0, 100.0]; + let crop = Crop::from_rect([50.0, -20.0, 100.0, 40.0]); + assert_eq!( + crop.fractions(frame).expect("overlaps"), + [0.5, 0.0, 0.5, 0.2] + ); + assert!( + Crop::from_rect([200.0, 0.0, 10.0, 10.0]) + .fractions(frame) + .is_err() + ); + } + + #[test] + fn a_pixel_rect_is_at_least_one_pixel_and_inside_the_frame() { + let rect = pixel_rect([0.999, 0.999, 0.0005, 0.0005], 10, 10); + assert_eq!((rect.x, rect.y, rect.width, rect.height), (9, 9, 1, 1)); + let whole = pixel_rect([0.0, 0.0, 1.0, 1.0], 7, 5); + assert_eq!((whole.width, whole.height), (7, 5)); + } + + #[test] + fn cropping_takes_the_rows_of_the_rect() { + let picture: Vec = (0..16).collect(); // 4 x 4 + let rect = PixelRect { + x: 1, + y: 2, + width: 2, + height: 2, + }; + assert_eq!(crop_luma(&picture, 4, rect), [9, 10, 13, 14]); + let bgra: Vec = (0..64).collect(); + assert_eq!( + crop_bgra(&bgra, 4, rect)[..8], + [36, 37, 38, 39, 40, 41, 42, 43] + ); + } + + #[test] + fn a_small_change_under_the_threshold_is_not_a_changed_pixel_but_is_in_the_mean() { + let found = delta(&[100, 100], &[110, 100], 2, 24); + assert_eq!(found.changed, 0); + assert_eq!(found.bounds, None); + assert_eq!(found.mean, 5.0); + assert_eq!(found.peak, 10); + } + + #[test] + fn the_box_of_change_covers_every_changed_pixel() { + let before = vec![0; 16]; + let mut after = vec![0; 16]; + after[1] = 255; // (1, 0) + after[14] = 255; // (2, 3) + let found = delta(&before, &after, 4, 24); + assert_eq!(found.bounds, Some((1, 0, 2, 3))); + } + + #[test] + fn uniform_patches_have_no_variance_and_noisy_ones_do() { + assert_eq!( + evenness(&[10, 20, 30, 255, 10, 20, 30, 255]).rgb, + [30, 20, 10] + ); + assert!(evenness(&[0, 0, 0, 255, 255, 255, 255, 255]).variance > 10_000.0); + } +} diff --git a/komorebi-recorder/src/analysis/schemas.rs b/komorebi-recorder/src/analysis/schemas.rs index 83587309..f2d5e344 100644 --- a/komorebi-recorder/src/analysis/schemas.rs +++ b/komorebi-recorder/src/analysis/schemas.rs @@ -15,12 +15,16 @@ use super::answers::DeletedData; use super::answers::DiffData; use super::answers::EventsData; use super::answers::StateData; +use super::blank::BlankData; use super::brief::BriefData; use super::changes::ChangesData; use super::code::CodeData; +use super::compare::CompareData; use super::envelope::Envelope; +use super::framediff::FrameDiffData; use super::history::HistoryData; use super::logq::LogqData; +use super::motion::MotionData; use super::overview::OverviewData; use super::search::SearchData; use super::sheet::SheetData; @@ -110,6 +114,26 @@ pub const ALL: &[Entry] = &[ about: "`sheet`: a contact sheet image and where each frame is on it", schema: of::, }, + Entry { + name: "motion/1", + about: "`motion`: activity segments of a display or a region of it, with peak share and brightness delta", + schema: of::, + }, + Entry { + name: "framediff/1", + about: "`framediff`: how two frames differ: changed share, brightness delta, box of change", + schema: of::, + }, + Entry { + name: "blank/1", + about: "`blank`: stretches where a region was blank (one colour) or frozen", + schema: of::, + }, + Entry { + name: "compare/1", + about: "`compare`: two recordings side by side with the difference of each measure", + schema: of::, + }, Entry { name: "annotation/1", about: "`annotate add|edit`: the annotation as it now reads", diff --git a/komorebi-recorder/src/analysis/sheet.rs b/komorebi-recorder/src/analysis/sheet.rs index 871cf87b..e97d0aaf 100644 --- a/komorebi-recorder/src/analysis/sheet.rs +++ b/komorebi-recorder/src/analysis/sheet.rs @@ -181,14 +181,14 @@ pub struct SheetData { pub skipped: Vec, } -struct Canvas { +pub(super) struct Canvas { width: u32, height: u32, rgba: Vec, } impl Canvas { - fn new(width: u32, height: u32) -> Self { + pub(super) fn new(width: u32, height: u32) -> Self { let mut rgba = Vec::with_capacity(width as usize * height as usize * 4); for _ in 0..width as usize * height as usize { rgba.extend_from_slice(&[BACKGROUND[0], BACKGROUND[1], BACKGROUND[2], 255]); @@ -200,7 +200,7 @@ impl Canvas { } } - fn put(&mut self, x: i64, y: i64, color: [u8; 3], alpha: f32) { + pub(super) fn put(&mut self, x: i64, y: i64, color: [u8; 3], alpha: f32) { if x < 0 || y < 0 || x >= i64::from(self.width) || y >= i64::from(self.height) { return; } @@ -220,7 +220,15 @@ impl Canvas { } } - fn border(&mut self, x: u32, y: u32, width: u32, height: u32, thickness: u32, color: [u8; 3]) { + pub(super) fn border( + &mut self, + x: u32, + y: u32, + width: u32, + height: u32, + thickness: u32, + color: [u8; 3], + ) { self.rectangle(x, y, width, thickness, color); self.rectangle(x, y + height - thickness, width, thickness, color); self.rectangle(x, y, thickness, height, color); @@ -267,7 +275,7 @@ impl Canvas { } } - fn save(&self, path: &Path) -> eyre::Result<()> { + pub(super) fn save(&self, path: &Path) -> eyre::Result<()> { if let Some(parent) = path.parent() { std::fs::create_dir_all(parent) .wrap_err_with(|| format!("could not create {}", parent.display()))?; diff --git a/komorebi-recorder/src/cli/mod.rs b/komorebi-recorder/src/cli/mod.rs index e1c986bd..56df491d 100644 --- a/komorebi-recorder/src/cli/mod.rs +++ b/komorebi-recorder/src/cli/mod.rs @@ -47,7 +47,11 @@ pub use query::LocateArgs; pub use query::RangeArgs; pub use query::SummaryArgs; pub use record::RecordArgs; +pub use video::BlankArgs; pub use video::ChangesArgs; +pub use video::CompareArgs; +pub use video::FramediffArgs; +pub use video::MotionArgs; pub use video::SheetArgs; pub use viewer_command::ViewArgs; @@ -116,6 +120,14 @@ pub enum Command { Trace(TraceArgs), /// When and where on each display the picture changed Changes(ChangesArgs), + /// When, and how hard, a display or a region of it was moving + Motion(MotionArgs), + /// How two frames differ: changed share, brightness delta, box of change, optional PNG + Framediff(FramediffArgs), + /// When a region was blank (one colour, such as white) or frozen + Blank(BlankArgs), + /// Two recordings side by side: frames, motion, findings, annotations + Compare(CompareArgs), /// A contact sheet image of frames, with the times, events and bug region marked Sheet(SheetArgs), /// Open a recording in a window: play and scrub the video with every event on a timeline @@ -189,6 +201,10 @@ impl Command { Self::Window(args) => args.json, Self::Changes(args) => args.json, Self::Sheet(args) => args.json, + Self::Motion(args) => args.json, + Self::Framediff(args) => args.json, + Self::Blank(args) => args.json, + Self::Compare(args) => args.json, Self::Annotate(args) => args.json(), Self::Annotations(args) => args.json, Self::Doctor(args) => args.json, @@ -251,6 +267,10 @@ pub fn run(command: Command) -> eyre::Result<()> { Command::Changes(args) => video::changes(&args), Command::View(args) => crate::viewer::run(args.session), Command::Sheet(args) => video::sheet(&args), + Command::Motion(args) => video::motion(&args), + Command::Framediff(args) => video::framediff(&args), + Command::Blank(args) => video::blank(&args), + Command::Compare(args) => video::compare(&args), Command::Annotate(args) => annotate::annotate(&args), Command::Annotations(args) => annotate::list(&args), Command::Reindex(args) => reindex(&args), diff --git a/komorebi-recorder/src/cli/video.rs b/komorebi-recorder/src/cli/video.rs index 6d89ee35..c6224ba1 100644 --- a/komorebi-recorder/src/cli/video.rs +++ b/komorebi-recorder/src/cli/video.rs @@ -4,6 +4,10 @@ //! |---|---|---| //! | `changes` | [`Analysis::changes`](crate::analysis::Analysis::changes) | `changes/1` | //! | `sheet` | [`Analysis::sheet`] | `sheet/1` | +//! | `motion` | [`Analysis::motion`] | `motion/1` | +//! | `framediff` | [`framediff`](crate::analysis::framediff::framediff) | `framediff/1` | +//! | `blank` | [`Analysis::blank`] | `blank/1` | +//! | `compare` | [`Analysis::compare`] | `compare/1` | //! //! Both work on a range of video time, given as `--from` and `--to` (any time `state --at` takes; //! the default is the whole video) or, instead of those, as `--bug `: a bug region from three @@ -18,9 +22,20 @@ use super::print_json; use crate::analysis::Analysis; +use crate::analysis::blank::DEFAULT_MIN_DURATION_S; +use crate::analysis::blank::DEFAULT_VARIANCE; +use crate::analysis::blank::SCHEMA_BLANK; use crate::analysis::changes::SCHEMA_CHANGES; +use crate::analysis::compare::SCHEMA_COMPARE; use crate::analysis::envelope; use crate::analysis::envelope::Envelope; +use crate::analysis::framediff::DEFAULT_WIDTH; +use crate::analysis::framediff::FrameAt; +use crate::analysis::framediff::FrameDiffOptions; +use crate::analysis::framediff::SCHEMA_FRAMEDIFF; +use crate::analysis::motion::SCHEMA_MOTION; +use crate::analysis::motion::ScanOptions; +use crate::analysis::pixels::Crop; use crate::analysis::sheet::Region; use crate::analysis::sheet::SCHEMA_SHEET; use crate::analysis::sheet::SheetOptions; @@ -96,6 +111,102 @@ pub struct SheetArgs { pub json: bool, } +/// The arguments of `motion`. +#[derive(Args, Debug)] +pub struct MotionArgs { + /// Session directory + pub session: PathBuf, + /// Only this display (repeatable) [default: every display with video] + #[arg(long = "display", value_name = "ID")] + pub displays: Vec, + /// Only this part of the display, in display points: x,y,w,h + #[arg(long, value_name = "X,Y,W,H")] + pub region: Option, + /// From this moment [default: the start of the video] + #[arg(long, value_name = "TIME")] + pub from: Option, + /// To this moment [default: the end of the video] + #[arg(long, value_name = "TIME")] + pub to: Option, + /// Above 1 notices smaller changes, below 1 only larger ones + #[arg(long, default_value_t = 1.0)] + pub sensitivity: f64, + /// Machine-readable JSON output + #[arg(long)] + pub json: bool, +} + +/// The arguments of `framediff`. +#[derive(Args, Debug)] +pub struct FramediffArgs { + /// Session directory + pub session: PathBuf, + /// The first moment + #[arg(value_name = "TIME_A")] + pub time_a: String, + /// The second moment [default with --against: the same as TIME_A] + #[arg(value_name = "TIME_B")] + pub time_b: Option, + /// Compare with the frame of this other recording at TIME_B (resolved in that recording) + #[arg(long, value_name = "SESSION")] + pub against: Option, + /// The display [default: the first display with video] + #[arg(long = "display", value_name = "ID")] + pub display: Option, + /// Only this part of the display, in display points: x,y,w,h + #[arg(long, value_name = "X,Y,W,H")] + pub region: Option, + /// Write a PNG of the compared area with the changed pixels in red + #[arg(long, value_name = "FILE")] + pub png: Option, + /// The width frames are decoded at, in pixels + #[arg(long, default_value_t = DEFAULT_WIDTH)] + pub width: u32, + /// Machine-readable JSON output + #[arg(long)] + pub json: bool, +} + +/// The arguments of `blank`. +#[derive(Args, Debug)] +pub struct BlankArgs { + /// Session directory + pub session: PathBuf, + /// The region to watch, in display points: x,y,w,h + #[arg(long, value_name = "X,Y,W,H")] + pub region: String, + /// Only this display (repeatable) [default: every display with video] + #[arg(long = "display", value_name = "ID")] + pub displays: Vec, + /// From this moment [default: the start of the video] + #[arg(long, value_name = "TIME")] + pub from: Option, + /// To this moment [default: the end of the video] + #[arg(long, value_name = "TIME")] + pub to: Option, + /// A region is blank when the variance of its brightness is below this + #[arg(long, default_value_t = DEFAULT_VARIANCE)] + pub variance: f64, + /// Drop stretches shorter than this many milliseconds + #[arg(long, value_name = "MS", default_value_t = DEFAULT_MIN_DURATION_S * 1000.0)] + pub min_ms: f64, + /// Machine-readable JSON output + #[arg(long)] + pub json: bool, +} + +/// The arguments of `compare`. +#[derive(Args, Debug)] +pub struct CompareArgs { + /// The first session directory + pub session_a: PathBuf, + /// The second session directory + pub session_b: PathBuf, + /// Machine-readable JSON output + #[arg(long)] + pub json: bool, +} + /// The video-time range to work on: a bug region, or `--from`/`--to`, or everything. fn range( analysis: &Analysis, @@ -233,3 +344,139 @@ pub(super) fn sheet(args: &SheetArgs) -> eyre::Result<()> { print!("{}", report::sheet_text(&data)); Ok(()) } + +fn region(text: Option<&str>) -> eyre::Result> { + text.map(Crop::parse).transpose() +} + +pub(super) fn motion(args: &MotionArgs) -> eyre::Result<()> { + let analysis = Analysis::open(&args.session)?; + let video = SessionVideo::new(&analysis.manifest, analysis.guard()); + let (from, to) = range(&analysis, args.from.as_deref(), args.to.as_deref(), None)?; + let shown = displays(&video, &args.displays)?; + + let data = analysis.motion( + &video, + &ScanOptions { + displays: &shown, + from_s: from, + to_s: to, + crop: region(args.region.as_deref())?, + cancel: None, + }, + args.sensitivity, + )?; + if args.json { + return print_json(&Envelope::ok(SCHEMA_MOTION, data).warnings(analysis.warnings())); + } + + print!("{}", data.to_text()); + Ok(()) +} + +pub(super) fn framediff(args: &FramediffArgs) -> eyre::Result<()> { + let analysis = Analysis::open(&args.session)?; + let video = SessionVideo::new(&analysis.manifest, analysis.guard()); + let wanted: Vec = args.display.into_iter().collect(); + let display = displays(&video, &wanted)?[0]; + let options = FrameDiffOptions { + crop: region(args.region.as_deref())?, + width: args.width, + png: args.png.as_deref(), + }; + let first = FrameAt { + analysis: &analysis, + video: &video, + display, + time_s: analysis.resolve(&args.time_a)?.video_s, + }; + + let data = if let Some(path) = &args.against { + let other = Analysis::open(path)?; + let other_video = SessionVideo::new(&other.manifest, other.guard()); + let available = other_video.displays(); + let other_display = if available.contains(&display) { + display + } else if let [only] = available[..] { + only + } else { + return Err(envelope::not_found(format!( + "{} has no display {display}; displays with video: {available:?}", + path.display() + ))); + }; + let time_b = other + .resolve(args.time_b.as_deref().unwrap_or(&args.time_a))? + .video_s; + crate::analysis::framediff::framediff( + &first, + &FrameAt { + analysis: &other, + video: &other_video, + display: other_display, + time_s: time_b, + }, + &options, + )? + } else { + let text = args.time_b.as_deref().ok_or_else(|| { + envelope::usage("framediff needs a second time, or --against another session") + })?; + let time_b = analysis.resolve(text)?.video_s; + crate::analysis::framediff::framediff( + &first, + &FrameAt { + time_s: time_b, + ..first + }, + &options, + )? + }; + if args.json { + return print_json(&Envelope::ok(SCHEMA_FRAMEDIFF, data).warnings(analysis.warnings())); + } + + print!("{}", data.to_text()); + Ok(()) +} + +pub(super) fn blank(args: &BlankArgs) -> eyre::Result<()> { + let analysis = Analysis::open(&args.session)?; + let video = SessionVideo::new(&analysis.manifest, analysis.guard()); + let (from, to) = range(&analysis, args.from.as_deref(), args.to.as_deref(), None)?; + let shown = displays(&video, &args.displays)?; + + let data = analysis.blank( + &video, + &ScanOptions { + displays: &shown, + from_s: from, + to_s: to, + crop: region(Some(&args.region))?, + cancel: None, + }, + args.variance, + args.min_ms / 1000.0, + )?; + if args.json { + return print_json(&Envelope::ok(SCHEMA_BLANK, data).warnings(analysis.warnings())); + } + + print!("{}", data.to_text()); + Ok(()) +} + +pub(super) fn compare(args: &CompareArgs) -> eyre::Result<()> { + let a = Analysis::open(&args.session_a)?; + let b = Analysis::open(&args.session_b)?; + + let data = a.compare(&b, None)?; + if args.json { + let mut warnings = a.warnings(); + warnings.extend(b.warnings()); + return print_json(&Envelope::ok(SCHEMA_COMPARE, data).warnings(warnings)); + } + + print!("{}", data.to_text()); + Ok(()) +} diff --git a/komorebi-recorder/src/guide.md b/komorebi-recorder/src/guide.md index d3396efb..8823ebf1 100644 --- a/komorebi-recorder/src/guide.md +++ b/komorebi-recorder/src/guide.md @@ -96,6 +96,11 @@ locations as hints. `unmanaged-window` (on screen and not managed: usually right, and komorebi's explanation is quoted), and `trace event:N` shows the facts. Without the probe none of that is knowable, and `overview` says nothing about it. +- To measure the picture instead of scripting `ffmpeg`: `motion` (when a display or `--region x,y,w,h` moved, + and how hard), `framediff ` (what changed between two moments; `--png` shows where), + `blank --region x,y,w,h` (when a region went one colour, such as white, or froze) and + `compare ` (two recordings' frames, motion and findings side by side). Regions are in display + points, the same as the `bbox_points` of `changes`. They need a host that can decode the video. - Video time `00:00.000` is the first frame of any display. A display that was unplugged, asleep or whose capture failed has a gap, and `incident` records say why. diff --git a/komorebi-recorder/tests/video.rs b/komorebi-recorder/tests/video.rs index 00f31cb9..7685a543 100644 --- a/komorebi-recorder/tests/video.rs +++ b/komorebi-recorder/tests/video.rs @@ -284,3 +284,183 @@ fn a_moment_a_display_had_no_video_is_skipped_not_invented() { ]); assert_eq!(huge.status.code(), Some(2)); } + +#[test] +fn motion_finds_the_segments_and_a_region_limits_where_it_looks() { + let Some(dir) = session("video-motion") else { + return; + }; + + let answer = json(&["motion", &dir, "--display", "1", "--json"]); + assert_eq!(answer["schema"], "motion/1"); + let segments = answer["data"]["segments"].as_array().expect("segments"); + assert_eq!(segments.len(), 2, "{segments:#?}"); + assert!(segments[0]["peak_share"].as_f64().expect("share") > 0.0); + assert!(segments[0]["peak_mean_delta"].as_f64().expect("delta") > 0.0); + let duration = segments[1]["duration_s"].as_f64().expect("duration"); + assert!((duration - 0.97).abs() < 0.1, "{duration}"); + assert_eq!(segments[1]["events"], json!([1])); + + // counting changes the blocks on the right of the picture: the left half of the display (the + // test display is 1000 x 800 points) holds still, the right half moves in both segments + let left = json(&[ + "motion", + &dir, + "--display", + "1", + "--region", + "0,0,500,800", + "--json", + ]); + assert_eq!(left["data"]["segments"], json!([])); + let right = json(&[ + "motion", + &dir, + "--display", + "1", + "--region", + "500,0,500,800", + "--json", + ]); + let moved = right["data"]["segments"].as_array().expect("segments"); + assert_eq!(moved.len(), 2, "{moved:#?}"); + assert!(moved[0]["bbox_points"][0].as_f64().expect("x") >= 500.0); + + // a region off the display is a usage error, so is a malformed one + for bad in ["5000,0,10,10", "1,2,3"] { + let output = recorder(&["motion", &dir, "--region", bad]); + assert_eq!(output.status.code(), Some(2), "{bad}"); + } +} + +#[test] +fn framediff_says_how_much_changed_and_where_and_writes_the_picture() { + let Some(dir) = session("video-framediff") else { + return; + }; + let png = std::path::Path::new(&dir).join("diff.png"); + let png_arg = png.display().to_string(); + + // the picture is counting at 0.2 s and 0.5 s + let moved = json(&[ + "framediff", + &dir, + "0.2", + "0.5", + "--display", + "1", + "--png", + &png_arg, + "--json", + ]); + assert_eq!(moved["schema"], "framediff/1"); + let data = &moved["data"]; + assert!(data["changed_share"].as_f64().expect("share") > 0.0); + assert!(data["peak_delta"].as_u64().expect("peak") > 100); + assert!(data["bbox"][0].as_f64().expect("x") >= 0.5); + assert!(png.is_file()); + assert_eq!(&std::fs::read(&png).expect("png")[1..4], b"PNG"); + + // it holds still from 1.1 s to 1.8 s, and a recording is the same as itself + let still = json(&["framediff", &dir, "1.1", "1.8", "--display", "1", "--json"]); + assert_eq!(still["data"]["changed_pixels"], 0); + assert!(still["data"]["bbox"].is_null()); + let same = json(&[ + "framediff", + &dir, + "0.5", + "--display", + "1", + "--against", + &dir, + "--json", + ]); + assert_eq!(same["data"]["changed_pixels"], 0); + + // without a second time or another session there is nothing to compare with + assert_eq!( + recorder(&["framediff", &dir, "0.5", "--display", "1"]) + .status + .code(), + Some(2) + ); +} + +#[test] +fn blank_finds_the_region_going_white_and_the_picture_freezing() { + if !testkit::ffmpeg_available() { + eprintln!("skipped: ffmpeg is not available to make a test video"); + return; + } + + // a still picture for a second, all white for half a second, the picture again + let (session, _) = SessionBuilder::new("video-blank") + .video_showing(1, 0.0, 90, 30, |frame| match frame { + 30..=44 => 0xFFFF, + _ => 0x5A5A, + }) + .build(); + let dir = session.dir.display().to_string(); + + let answer = json(&["blank", &dir, "--region", "0,0,1000,800", "--json"]); + assert_eq!(answer["schema"], "blank/1"); + let stretches = answer["data"]["stretches"].as_array().expect("stretches"); + let blank: Vec<&Value> = stretches + .iter() + .filter(|stretch| stretch["kind"] == "blank") + .collect(); + assert_eq!(blank.len(), 1, "{stretches:#?}"); + assert_eq!(blank[0]["colour"], "white"); + assert!((blank[0]["start_video_s"].as_f64().expect("start") - 1.0).abs() < 0.06); + assert!((blank[0]["duration_s"].as_f64().expect("duration") - 0.5).abs() < 0.1); + + // the picture before and after repeats, frame after frame + let frozen = stretches + .iter() + .filter(|stretch| stretch["kind"] == "frozen") + .count(); + assert!(frozen >= 2, "{stretches:#?}"); + + // a minimum length drops what is shorter, and a region is required + let long = json(&[ + "blank", + &dir, + "--region", + "0,0,1000,800", + "--min-ms", + "5000", + "--json", + ]); + assert_eq!(long["data"]["stretches"], json!([])); + assert_eq!(recorder(&["blank", &dir]).status.code(), Some(2)); +} + +#[test] +fn compare_sets_two_recordings_side_by_side() { + let Some(dir) = session("video-compare") else { + return; + }; + // frames come every 33 ms, except for a hole of 300 ms + let mut csv = String::from("index,pts_mach_ns,received_mach_ns\n"); + let mut at = 1_000_000_000_u64; + for index in 0..60 { + csv.push_str(&format!("{index},{at},{at}\n")); + at += if index == 29 { 300_000_000 } else { 33_000_000 }; + } + std::fs::write(std::path::Path::new(&dir).join("frames-1.csv"), csv).expect("frame log"); + + let answer = json(&["compare", &dir, &dir, "--json"]); + assert_eq!(answer["schema"], "compare/1"); + let data = &answer["data"]; + assert_eq!(data["a"]["frames"], 60); + assert!((data["a"]["worst_gap_ms"].as_f64().expect("gap") - 300.0).abs() < 1.0); + assert_eq!(data["a"]["late_gaps"], 1); + // two on display 1 and one on display 2 + assert_eq!(data["a"]["motion"]["segments"], 3); + assert_eq!(data["deltas"]["frames"], 0); + assert_eq!(data["deltas"]["worst_gap_ms"], 0.0); + + let text = recorder(&["compare", &dir, &dir]); + assert!(text.status.success()); + assert!(String::from_utf8_lossy(&text.stdout).contains("worst frame gap")); +}