mirror of
https://github.com/LGUG2Z/komorebi.git
synced 2026-11-17 14:21:22 +01:00
feat(video): motion, framediff, blank and compare for recordings
Built-in frame analysis for agents that scripted ffmpeg tblend/signalstats. motion is changes with an optional --region (display points) and a peak mean luma delta; framediff compares two frames (or one moment of two recordings with --against) with an optional PNG; blank reports stretches where a region is near-uniform (named colour) or frozen; compare sets two recordings side by side. changes now shares the engine with motion and behaves as before. The crop API takes a rect so a window rect can be passed later. No macOS-specific code touched (not buildable here).
This commit is contained in:
1 parent
1f6e4cef05
commit
6af9eabd27
14 files changed
+2942
-109
No files matched your search
@@ -119,6 +119,10 @@ and `event:12.end` when it finished.
|
||||
| `search TEXT` | Where a word appears: komorebi's log, events, incidents, annotations (and `--oslog` for the macOS trace) |
|
||||
| `changes` | When and where on each display the picture changed, and which events fall inside |
|
||||
| `sheet` | A contact sheet image of frames with times, events and the bug region marked |
|
||||
| `motion [--region x,y,w,h]` | When, and how hard, a display or a region of it was moving: segments with start, duration, frame count, peak share of pixels changed and peak mean brightness delta (replaces scripting ffmpeg `tblend`) |
|
||||
| `framediff A B [--region ..] [--png FILE] [--against SESSION]` | How two frames differ: changed share, mean and peak brightness delta, the box of change, and a PNG with the changed pixels in red |
|
||||
| `blank --region x,y,w,h` | When a region was blank (one colour, named: `white`, `black`, a hex code) or frozen (identical to the frame before): "the window went white for 200 ms" |
|
||||
| `compare A B` | Two recordings side by side: duration, frames, effective fps, worst frame gap, motion, findings by kind, annotations, and the difference |
|
||||
| `annotate`, `annotations` | Add, change and list bug regions, notes and findings |
|
||||
|
||||
### Who changed it
|
||||
|
||||
@@ -0,0 +1,515 @@
|
||||
//! `blank`: when a region of a display was a blank patch of one colour, or frozen.
|
||||
//!
|
||||
//! The question behind it is "did the content of this window go white for a moment?". Every frame
|
||||
//! of the range is cut to the region and measured, and two kinds of stretch are reported:
|
||||
//!
|
||||
//! - **blank**: the region is near-uniform, its brightness varying less than a threshold
|
||||
//! (variance of the 0 to 255 brightness, default 4.0: a standard deviation of 2), and the colour
|
||||
//! is named (`white`, `black`, `gray`, or a hex code). A run of blank frames of one colour is one
|
||||
//! stretch, which lasts until the next frame replaced it.
|
||||
//! - **frozen**: consecutive frames in which the region is identical (no pixel differs by more than
|
||||
//! 2 of 255). The video only has a frame when something on the screen changed, so a frozen
|
||||
//! region is one that stayed the same while something else on the display kept changing: a
|
||||
//! stretch from the frame that showed it to the last repeat. A display that held entirely
|
||||
//! still has no frames and so no frozen stretch.
|
||||
//!
|
||||
//! Stretches shorter than `min_duration_s` are dropped. A stretch is judged on frames decoded at a
|
||||
//! size that gives the region 240 pixels across, so a few stray pixels in a large region are
|
||||
//! averaged away; use a smaller region to see them.
|
||||
|
||||
use super::Analysis;
|
||||
use super::changes::Gap;
|
||||
use super::envelope;
|
||||
use super::motion::ScanOptions;
|
||||
use super::motion::scale_for;
|
||||
use super::pixels;
|
||||
use super::pixels::Crop;
|
||||
use super::video::SessionVideo;
|
||||
use crate::av::Scale;
|
||||
use color_eyre::eyre;
|
||||
use schemars::JsonSchema;
|
||||
use serde::Serialize;
|
||||
|
||||
/// The schema name that identifies a [`BlankData`] in an envelope (`blank/1`).
|
||||
pub const SCHEMA_BLANK: &str = "blank/1";
|
||||
|
||||
/// The default brightness variance under which a region is blank.
|
||||
pub const DEFAULT_VARIANCE: f64 = 4.0;
|
||||
|
||||
/// The default shortest stretch reported, in seconds.
|
||||
pub const DEFAULT_MIN_DURATION_S: f64 = 0.0;
|
||||
|
||||
/// A pixel that moved by no more than this (of 255) is the same pixel.
|
||||
const FROZEN_TOLERANCE: i32 = 2;
|
||||
|
||||
/// What a [`Stretch`] is.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, JsonSchema)]
|
||||
#[serde(rename_all = "lowercase")]
|
||||
pub enum StretchKind {
|
||||
/// The region was one colour.
|
||||
Blank,
|
||||
/// The region was identical in consecutive frames.
|
||||
Frozen,
|
||||
}
|
||||
|
||||
/// A stretch of video in which the region was blank, or frozen.
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, JsonSchema)]
|
||||
pub struct Stretch {
|
||||
/// The id of the display.
|
||||
pub display: u32,
|
||||
/// Whether the region was blank or frozen.
|
||||
pub kind: StretchKind,
|
||||
/// When the stretch began, in seconds of video time.
|
||||
pub start_video_s: f64,
|
||||
/// When it ended, in seconds of video time: when a frame showed something else (a blank stretch
|
||||
/// that lasted to the end of the range ends there), or the last repeat for a frozen one.
|
||||
pub end_video_s: f64,
|
||||
/// `end_video_s - start_video_s`.
|
||||
pub duration_s: f64,
|
||||
/// For a blank stretch the frames that were blank; for a frozen one the frames that repeated
|
||||
/// the one before.
|
||||
pub frames: usize,
|
||||
/// For a blank stretch, the colour: `white`, `black`, `gray` or a hex code. `None` when frozen.
|
||||
pub colour: Option<String>,
|
||||
/// For a blank stretch, the mean colour `[red, green, blue]`. `None` when frozen.
|
||||
pub rgb: Option<[u8; 3]>,
|
||||
}
|
||||
|
||||
/// Where a region was blank or frozen over a range: the `data` of `blank`.
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, JsonSchema)]
|
||||
pub struct BlankData {
|
||||
/// The start of the range scanned, in seconds of video time.
|
||||
pub from_video_s: f64,
|
||||
/// The end of the range scanned, in seconds of video time.
|
||||
pub to_video_s: f64,
|
||||
/// The displays scanned.
|
||||
pub displays: Vec<u32>,
|
||||
/// The region, in global display points; `None` is the whole display.
|
||||
pub region: Option<Crop>,
|
||||
/// A region is blank when the variance of its brightness is below this.
|
||||
pub variance_threshold: f64,
|
||||
/// Stretches shorter than this were dropped, in seconds.
|
||||
pub min_duration_s: f64,
|
||||
/// How many frames were decoded and measured, over all displays.
|
||||
pub frames_examined: usize,
|
||||
/// The stretches, ordered by start, then display, then kind.
|
||||
pub stretches: Vec<Stretch>,
|
||||
/// Stretches of the range where a display had no video at all.
|
||||
pub gaps: Vec<Gap>,
|
||||
}
|
||||
|
||||
impl BlankData {
|
||||
/// Renders the answer as text, one line for each stretch.
|
||||
///
|
||||
/// # Example
|
||||
///
|
||||
/// ```rust
|
||||
/// use komorebi_recorder::analysis::blank::{BlankData, Stretch, StretchKind};
|
||||
///
|
||||
/// let data = BlankData {
|
||||
/// from_video_s: 0.0,
|
||||
/// to_video_s: 6.0,
|
||||
/// displays: vec![1],
|
||||
/// region: None,
|
||||
/// variance_threshold: 4.0,
|
||||
/// min_duration_s: 0.0,
|
||||
/// frames_examined: 50,
|
||||
/// stretches: vec![Stretch {
|
||||
/// display: 1,
|
||||
/// kind: StretchKind::Blank,
|
||||
/// start_video_s: 3.28,
|
||||
/// end_video_s: 3.5,
|
||||
/// duration_s: 0.22,
|
||||
/// frames: 3,
|
||||
/// colour: Some("white".to_string()),
|
||||
/// rgb: Some([255, 255, 255]),
|
||||
/// }],
|
||||
/// gaps: vec![],
|
||||
/// };
|
||||
/// assert!(data.to_text().contains("blank (white)"));
|
||||
/// ```
|
||||
#[must_use]
|
||||
pub fn to_text(&self) -> String {
|
||||
use crate::timecode::format as time;
|
||||
use std::fmt::Write;
|
||||
|
||||
let mut out = String::new();
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"{} frames measured on display(s) {} from {} to {}: {} stretch(es) blank or frozen",
|
||||
self.frames_examined,
|
||||
self.displays
|
||||
.iter()
|
||||
.map(ToString::to_string)
|
||||
.collect::<Vec<_>>()
|
||||
.join(", "),
|
||||
time(self.from_video_s),
|
||||
time(self.to_video_s),
|
||||
self.stretches.len(),
|
||||
);
|
||||
for stretch in &self.stretches {
|
||||
let what = match (&stretch.kind, &stretch.colour) {
|
||||
(StretchKind::Blank, Some(colour)) => format!("blank ({colour})"),
|
||||
(StretchKind::Blank, None) => "blank".to_string(),
|
||||
(StretchKind::Frozen, _) => "frozen".to_string(),
|
||||
};
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"display {} {} - {} ({:.0} ms, {} frames) {what}",
|
||||
stretch.display,
|
||||
time(stretch.start_video_s),
|
||||
time(stretch.end_video_s),
|
||||
stretch.duration_s * 1000.0,
|
||||
stretch.frames,
|
||||
);
|
||||
}
|
||||
for gap in &self.gaps {
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"display {} has no video from {} to {}",
|
||||
gap.display,
|
||||
time(gap.from_video_s),
|
||||
time(gap.to_video_s)
|
||||
);
|
||||
}
|
||||
out
|
||||
}
|
||||
}
|
||||
|
||||
/// What one frame showed in the region.
|
||||
#[derive(Debug, Clone, Copy, PartialEq)]
|
||||
pub struct Sample {
|
||||
/// When the frame was shown, in seconds of video time.
|
||||
pub shown_s: f64,
|
||||
/// The variance of the region's brightness.
|
||||
pub variance: f64,
|
||||
/// The mean colour of the region.
|
||||
pub rgb: [u8; 3],
|
||||
/// Whether the region is identical to what the previous frame showed.
|
||||
pub identical: bool,
|
||||
}
|
||||
|
||||
/// Finds the blank and frozen stretches in the samples of one display, which are in time order.
|
||||
/// `to_s` is where the range ends: a blank stretch still going at the last sample lasts to it.
|
||||
///
|
||||
/// # Example
|
||||
///
|
||||
/// ```rust
|
||||
/// use komorebi_recorder::analysis::blank::{Sample, StretchKind, stretches};
|
||||
///
|
||||
/// let white = |shown_s, identical| Sample { shown_s, variance: 0.0, rgb: [255, 255, 255], identical };
|
||||
/// let busy = |shown_s| Sample { shown_s, variance: 900.0, rgb: [90, 80, 70], identical: false };
|
||||
///
|
||||
/// // busy until 1.0 s, white from 1.0 s to 1.4 s, busy again
|
||||
/// let found = stretches(1, &[busy(0.5), white(1.0, false), white(1.2, true), busy(1.4)], 2.0, 4.0, 0.0);
|
||||
/// assert_eq!(found.len(), 2);
|
||||
/// assert_eq!(found[0].kind, StretchKind::Blank);
|
||||
/// assert_eq!(found[0].colour.as_deref(), Some("white"));
|
||||
/// assert!((found[0].duration_s - 0.4).abs() < 1e-9);
|
||||
/// assert_eq!(found[1].kind, StretchKind::Frozen);
|
||||
/// ```
|
||||
#[must_use]
|
||||
pub fn stretches(
|
||||
display: u32,
|
||||
samples: &[Sample],
|
||||
to_s: f64,
|
||||
variance: f64,
|
||||
min_duration_s: f64,
|
||||
) -> Vec<Stretch> {
|
||||
let mut found = Vec::new();
|
||||
|
||||
// blank: runs of blank frames of one colour
|
||||
let mut index = 0;
|
||||
while index < samples.len() {
|
||||
let sample = &samples[index];
|
||||
if sample.variance >= variance {
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
let name = pixels::colour_name(sample.rgb);
|
||||
let mut last = index;
|
||||
while samples
|
||||
.get(last + 1)
|
||||
.is_some_and(|next| next.variance < variance && pixels::colour_name(next.rgb) == name)
|
||||
{
|
||||
last += 1;
|
||||
}
|
||||
|
||||
let run = &samples[index..=last];
|
||||
let end = samples
|
||||
.get(last + 1)
|
||||
.map_or(to_s.max(samples[last].shown_s), |next| next.shown_s);
|
||||
let mean = |channel: usize| {
|
||||
(run.iter().map(|s| f64::from(s.rgb[channel])).sum::<f64>() / run.len() as f64).round()
|
||||
as u8
|
||||
};
|
||||
found.push(Stretch {
|
||||
display,
|
||||
kind: StretchKind::Blank,
|
||||
start_video_s: sample.shown_s,
|
||||
end_video_s: end,
|
||||
duration_s: end - sample.shown_s,
|
||||
frames: run.len(),
|
||||
colour: Some(name),
|
||||
rgb: Some([mean(0), mean(1), mean(2)]),
|
||||
});
|
||||
index = last + 1;
|
||||
}
|
||||
|
||||
// frozen: runs of frames identical to the one before, from that one
|
||||
let mut index = 1;
|
||||
while index < samples.len() {
|
||||
if !samples[index].identical {
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
let mut last = index;
|
||||
while samples.get(last + 1).is_some_and(|next| next.identical) {
|
||||
last += 1;
|
||||
}
|
||||
let (start, end) = (samples[index - 1].shown_s, samples[last].shown_s);
|
||||
found.push(Stretch {
|
||||
display,
|
||||
kind: StretchKind::Frozen,
|
||||
start_video_s: start,
|
||||
end_video_s: end,
|
||||
duration_s: end - start,
|
||||
frames: last - index + 1,
|
||||
colour: None,
|
||||
rgb: None,
|
||||
});
|
||||
index = last + 1;
|
||||
}
|
||||
|
||||
found.retain(|stretch| stretch.duration_s >= min_duration_s);
|
||||
found.sort_by(|a, b| {
|
||||
a.start_video_s
|
||||
.total_cmp(&b.start_video_s)
|
||||
.then((a.kind as u8).cmp(&(b.kind as u8)))
|
||||
});
|
||||
found
|
||||
}
|
||||
|
||||
impl Analysis {
|
||||
/// Finds where the region of `options` (the whole display if it has none) was blank, or
|
||||
/// frozen, on each display, as the [module](self) describes.
|
||||
///
|
||||
/// `variance` is the brightness variance under which a region is blank (see
|
||||
/// [`DEFAULT_VARIANCE`]); stretches shorter than `min_duration_s` are dropped. Stops early,
|
||||
/// without an error, once `options.cancel` is set.
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// Returns an error if `variance` is negative or `min_duration_s` is, or the region does not
|
||||
/// overlap a display or a display's frame is unknown (the usage kind), or a video file cannot
|
||||
/// be resolved, opened or decoded, or the host has no decoder.
|
||||
///
|
||||
/// # Example
|
||||
///
|
||||
/// This needs recorded video, so it is not run here; `tests/video.rs` runs it through the
|
||||
/// `blank` command.
|
||||
///
|
||||
/// ```rust,no_run
|
||||
/// use komorebi_recorder::analysis::Analysis;
|
||||
/// use komorebi_recorder::analysis::blank::{DEFAULT_MIN_DURATION_S, DEFAULT_VARIANCE};
|
||||
/// use komorebi_recorder::analysis::motion::ScanOptions;
|
||||
/// use komorebi_recorder::analysis::pixels::Crop;
|
||||
/// use komorebi_recorder::analysis::video::SessionVideo;
|
||||
///
|
||||
/// let analysis = Analysis::open(std::path::Path::new("session-dir"))?;
|
||||
/// let video = SessionVideo::new(&analysis.manifest, analysis.guard());
|
||||
/// let data = analysis.blank(
|
||||
/// &video,
|
||||
/// &ScanOptions {
|
||||
/// displays: &video.displays(),
|
||||
/// from_s: 0.0,
|
||||
/// to_s: 60.0,
|
||||
/// crop: Some(Crop::parse("500,300,1200,800")?),
|
||||
/// cancel: None,
|
||||
/// },
|
||||
/// DEFAULT_VARIANCE,
|
||||
/// DEFAULT_MIN_DURATION_S,
|
||||
/// )?;
|
||||
/// print!("{}", data.to_text());
|
||||
/// # Ok::<(), Box<dyn std::error::Error>>(())
|
||||
/// ```
|
||||
pub fn blank(
|
||||
&self,
|
||||
video: &SessionVideo,
|
||||
options: &ScanOptions,
|
||||
variance: f64,
|
||||
min_duration_s: f64,
|
||||
) -> eyre::Result<BlankData> {
|
||||
if variance < 0.0 || min_duration_s < 0.0 {
|
||||
return Err(envelope::usage(
|
||||
"--variance and --min-ms cannot be negative",
|
||||
));
|
||||
}
|
||||
|
||||
let mut found = Vec::new();
|
||||
let mut examined = 0;
|
||||
for display in options.displays {
|
||||
let (crop, scale) = match &options.crop {
|
||||
Some(crop) => {
|
||||
let fractions = self.crop_fractions(*display, crop)?;
|
||||
(Some(fractions), scale_for(fractions[2]))
|
||||
}
|
||||
None => (None, Scale::Width(240)),
|
||||
};
|
||||
|
||||
let mut samples: Vec<Sample> = Vec::new();
|
||||
let mut previous: Option<(Vec<u8>, u32, u32)> = None;
|
||||
examined += video.scan(
|
||||
*display,
|
||||
options.from_s,
|
||||
options.to_s,
|
||||
scale,
|
||||
&mut |decoded| {
|
||||
if options
|
||||
.cancel
|
||||
.is_some_and(|cancel| cancel.load(std::sync::atomic::Ordering::Relaxed))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
let (width, height) = (decoded.frame.width, decoded.frame.height);
|
||||
let rect = crop.map(|crop| pixels::pixel_rect(crop, width, height));
|
||||
let (patch, row) = match rect {
|
||||
Some(rect) => (
|
||||
pixels::crop_bgra(&decoded.frame.bgra, width, rect),
|
||||
rect.width,
|
||||
),
|
||||
None => (decoded.frame.bgra.clone(), width),
|
||||
};
|
||||
let now = pixels::luma(&patch);
|
||||
let identical = previous.as_ref().is_some_and(|(before, w, h)| {
|
||||
(*w, *h) == (width, height)
|
||||
&& pixels::delta(before, &now, row, FROZEN_TOLERANCE).changed == 0
|
||||
});
|
||||
let measured = pixels::evenness(&patch);
|
||||
samples.push(Sample {
|
||||
shown_s: decoded.shown_s,
|
||||
variance: measured.variance,
|
||||
rgb: measured.rgb,
|
||||
identical,
|
||||
});
|
||||
previous = Some((now, width, height));
|
||||
true
|
||||
},
|
||||
)?;
|
||||
|
||||
found.extend(stretches(
|
||||
*display,
|
||||
&samples,
|
||||
options.to_s,
|
||||
variance,
|
||||
min_duration_s,
|
||||
));
|
||||
}
|
||||
|
||||
found.sort_by(|a, b| {
|
||||
a.start_video_s
|
||||
.total_cmp(&b.start_video_s)
|
||||
.then(a.display.cmp(&b.display))
|
||||
});
|
||||
|
||||
Ok(BlankData {
|
||||
from_video_s: options.from_s,
|
||||
to_video_s: options.to_s,
|
||||
displays: options.displays.to_vec(),
|
||||
region: options.crop,
|
||||
variance_threshold: variance,
|
||||
min_duration_s,
|
||||
frames_examined: examined,
|
||||
stretches: found,
|
||||
gaps: self.video_gaps(options.displays, options.from_s, options.to_s),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn sample(shown_s: f64, variance: f64, rgb: [u8; 3], identical: bool) -> Sample {
|
||||
Sample {
|
||||
shown_s,
|
||||
variance,
|
||||
rgb,
|
||||
identical,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_blank_stretch_lasts_until_the_frame_that_replaced_it() {
|
||||
let samples = [
|
||||
sample(0.0, 800.0, [90, 90, 90], false),
|
||||
sample(1.0, 0.5, [250, 250, 250], false),
|
||||
sample(1.5, 0.5, [250, 250, 250], false),
|
||||
sample(2.0, 800.0, [90, 90, 90], false),
|
||||
];
|
||||
let found = stretches(7, &samples, 5.0, 4.0, 0.0);
|
||||
assert_eq!(found.len(), 1);
|
||||
assert_eq!((found[0].start_video_s, found[0].end_video_s), (1.0, 2.0));
|
||||
assert_eq!(found[0].frames, 2);
|
||||
assert_eq!(found[0].display, 7);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_blank_stretch_at_the_end_lasts_to_the_end_of_the_range() {
|
||||
let samples = [
|
||||
sample(0.0, 800.0, [9; 3], false),
|
||||
sample(4.0, 0.0, [0; 3], false),
|
||||
];
|
||||
let found = stretches(1, &samples, 6.0, 4.0, 0.0);
|
||||
assert_eq!(found[0].colour.as_deref(), Some("black"));
|
||||
assert_eq!(found[0].end_video_s, 6.0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_change_of_blank_colour_is_two_stretches() {
|
||||
let samples = [
|
||||
sample(0.0, 0.0, [255; 3], false),
|
||||
sample(1.0, 0.0, [0; 3], false),
|
||||
];
|
||||
let found = stretches(1, &samples, 2.0, 4.0, 0.0);
|
||||
let colours: Vec<_> = found.iter().filter_map(|s| s.colour.as_deref()).collect();
|
||||
assert_eq!(colours, ["white", "black"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn frozen_runs_start_at_the_frame_that_was_repeated() {
|
||||
let samples = [
|
||||
sample(0.0, 800.0, [90; 3], false),
|
||||
sample(0.1, 800.0, [90; 3], false),
|
||||
sample(0.2, 800.0, [90; 3], true),
|
||||
sample(0.3, 800.0, [90; 3], true),
|
||||
sample(0.4, 800.0, [90; 3], false),
|
||||
];
|
||||
let found = stretches(1, &samples, 1.0, 4.0, 0.0);
|
||||
assert_eq!(found.len(), 1);
|
||||
assert_eq!(found[0].kind, StretchKind::Frozen);
|
||||
assert_eq!((found[0].start_video_s, found[0].end_video_s), (0.1, 0.3));
|
||||
assert_eq!(found[0].frames, 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn short_stretches_are_dropped() {
|
||||
let samples = [
|
||||
sample(0.0, 0.0, [255; 3], false),
|
||||
sample(0.05, 800.0, [90; 3], false),
|
||||
];
|
||||
assert!(stretches(1, &samples, 1.0, 4.0, 0.1).is_empty());
|
||||
assert_eq!(stretches(1, &samples, 1.0, 4.0, 0.0).len(), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_synthetic_white_patch_is_measured_as_blank_white() {
|
||||
let white = vec![255_u8; 4 * 4 * 4];
|
||||
let measured = pixels::evenness(&white);
|
||||
assert!(measured.variance < DEFAULT_VARIANCE);
|
||||
assert_eq!(pixels::colour_name(measured.rgb), "white");
|
||||
}
|
||||
}
|
||||
@@ -37,7 +37,7 @@
|
||||
|
||||
use super::Analysis;
|
||||
use super::envelope;
|
||||
use super::video::DecodedFrame;
|
||||
use super::pixels;
|
||||
use super::video::SessionVideo;
|
||||
use crate::av::Scale;
|
||||
use color_eyre::eyre;
|
||||
@@ -52,10 +52,10 @@ pub const SCHEMA_CHANGES: &str = "changes/1";
|
||||
const COMPARE_WIDTH: u32 = 240;
|
||||
|
||||
/// A pixel counts as changed if its brightness moved by more than this (out of 255).
|
||||
const PIXEL_THRESHOLD: i32 = 24;
|
||||
pub(super) const PIXEL_THRESHOLD: i32 = 24;
|
||||
|
||||
/// A frame is a change if at least this share of its pixels changed, at sensitivity 1.
|
||||
const SHARE_THRESHOLD: f64 = 0.0005;
|
||||
pub(super) const SHARE_THRESHOLD: f64 = 0.0005;
|
||||
|
||||
/// Frames of one change closer together than this are one change.
|
||||
const MERGE_GAP_S: f64 = 0.15;
|
||||
@@ -127,28 +127,176 @@ pub struct ChangesData {
|
||||
pub gaps: Vec<Gap>,
|
||||
}
|
||||
|
||||
/// The brightness of every pixel of a BGRA frame.
|
||||
fn luma(frame: &DecodedFrame) -> Vec<u8> {
|
||||
frame
|
||||
.frame
|
||||
.bgra
|
||||
.as_chunks::<4>()
|
||||
.0
|
||||
.iter()
|
||||
.map(|pixel| {
|
||||
// Rec. 601 weights in integers
|
||||
((u32::from(pixel[2]) * 299 + u32::from(pixel[1]) * 587 + u32::from(pixel[0]) * 114)
|
||||
/ 1000) as u8
|
||||
})
|
||||
.collect()
|
||||
/// One stretch of change being built: shared by `changes` and `motion`.
|
||||
pub(super) struct Open {
|
||||
pub(super) start: f64,
|
||||
pub(super) end: f64,
|
||||
pub(super) frames: usize,
|
||||
pub(super) peak: f64,
|
||||
/// The largest mean brightness difference between two consecutive frames in the stretch.
|
||||
pub(super) peak_delta: f64,
|
||||
/// `(min_x, min_y, max_x, max_y)` in pixels of the decoded frame.
|
||||
pub(super) bounds: (u32, u32, u32, u32),
|
||||
}
|
||||
|
||||
struct Open {
|
||||
start: f64,
|
||||
end: f64,
|
||||
frames: usize,
|
||||
peak: f64,
|
||||
/// Groups changed frames into stretches: frames less than [`MERGE_GAP_S`] apart are one.
|
||||
#[derive(Default)]
|
||||
pub(super) struct Segments {
|
||||
open: Option<Open>,
|
||||
finished: Vec<Open>,
|
||||
}
|
||||
|
||||
impl Segments {
|
||||
/// Adds a changed frame shown at `shown_s`, `share` of whose compared pixels changed, whose
|
||||
/// mean brightness difference from the frame before was `delta` and whose changed pixels lie
|
||||
/// in `bounds`.
|
||||
pub(super) fn push(
|
||||
&mut self,
|
||||
shown_s: f64,
|
||||
share: f64,
|
||||
delta: f64,
|
||||
bounds: (u32, u32, u32, u32),
|
||||
) {
|
||||
match &mut self.open {
|
||||
Some(span) if shown_s - span.end <= MERGE_GAP_S => {
|
||||
span.end = shown_s;
|
||||
span.frames += 1;
|
||||
span.peak = span.peak.max(share);
|
||||
span.peak_delta = span.peak_delta.max(delta);
|
||||
span.bounds = (
|
||||
span.bounds.0.min(bounds.0),
|
||||
span.bounds.1.min(bounds.1),
|
||||
span.bounds.2.max(bounds.2),
|
||||
span.bounds.3.max(bounds.3),
|
||||
);
|
||||
}
|
||||
_ => {
|
||||
self.finished.extend(self.open.take());
|
||||
self.open = Some(Open {
|
||||
start: shown_s,
|
||||
end: shown_s,
|
||||
frames: 1,
|
||||
peak: share,
|
||||
peak_delta: delta,
|
||||
bounds,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// The stretches, in order, including the one still open.
|
||||
pub(super) fn finish(mut self) -> Vec<Open> {
|
||||
self.finished.extend(self.open.take());
|
||||
self.finished
|
||||
}
|
||||
}
|
||||
|
||||
/// How [`track`] looks for change.
|
||||
pub(super) struct Tracking<'a> {
|
||||
/// The share of the compared pixels that must change for a frame to count.
|
||||
pub(super) threshold: f64,
|
||||
/// Only compare this part of the frame, as fractions `[x, y, width, height]` of it.
|
||||
pub(super) crop: Option<[f64; 4]>,
|
||||
/// What size to decode at.
|
||||
pub(super) scale: Scale,
|
||||
/// Stop early once this is set.
|
||||
pub(super) cancel: Option<&'a std::sync::atomic::AtomicBool>,
|
||||
}
|
||||
|
||||
/// What [`track`] found on one display.
|
||||
pub(super) struct Tracked {
|
||||
pub(super) spans: Vec<Open>,
|
||||
/// The size of the last frame decoded.
|
||||
pub(super) dimensions: Option<(u32, u32)>,
|
||||
pub(super) examined: usize,
|
||||
}
|
||||
|
||||
/// Decodes `display` over `from_s..=to_s` and finds the stretches in which it changed: the engine
|
||||
/// of `changes` and `motion`, described in the [module](self) documentation.
|
||||
pub(super) fn track(
|
||||
video: &SessionVideo,
|
||||
display: u32,
|
||||
from_s: f64,
|
||||
to_s: f64,
|
||||
how: &Tracking,
|
||||
) -> eyre::Result<Tracked> {
|
||||
let mut previous: Option<(Vec<u8>, u32, u32)> = None;
|
||||
let mut segments = Segments::default();
|
||||
|
||||
let examined = video.scan(display, from_s, to_s, how.scale, &mut |decoded| {
|
||||
if how
|
||||
.cancel
|
||||
.is_some_and(|cancel| cancel.load(std::sync::atomic::Ordering::Relaxed))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
let (width, height) = (decoded.frame.width, decoded.frame.height);
|
||||
let whole = pixels::luma(&decoded.frame.bgra);
|
||||
let rect = how.crop.map(|crop| pixels::pixel_rect(crop, width, height));
|
||||
let now = match rect {
|
||||
Some(rect) => pixels::crop_luma(&whole, width, rect),
|
||||
None => whole,
|
||||
};
|
||||
let (row, origin) = rect.map_or((width, (0, 0)), |rect| (rect.width, (rect.x, rect.y)));
|
||||
|
||||
if let Some((before, before_width, before_height)) = &previous
|
||||
&& (*before_width, *before_height) == (width, height)
|
||||
{
|
||||
let found = pixels::delta(before, &now, row, PIXEL_THRESHOLD);
|
||||
let share = found.share();
|
||||
if share >= how.threshold {
|
||||
let (min_x, min_y, max_x, max_y) = found.bounds.unwrap_or((0, 0, 0, 0));
|
||||
let (min_x, min_y, max_x, max_y) = (
|
||||
min_x + origin.0,
|
||||
min_y + origin.1,
|
||||
max_x + origin.0,
|
||||
max_y + origin.1,
|
||||
);
|
||||
segments.push(
|
||||
decoded.shown_s,
|
||||
share,
|
||||
found.mean,
|
||||
(min_x, min_y, max_x, max_y),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
previous = Some((now, width, height));
|
||||
true
|
||||
})?;
|
||||
|
||||
Ok(Tracked {
|
||||
spans: segments.finish(),
|
||||
dimensions: previous.map(|(_, width, height)| (width, height)),
|
||||
examined,
|
||||
})
|
||||
}
|
||||
|
||||
/// The box `bounds` (pixels) as fractions of a `dimensions` frame: `[x, y, width, height]`.
|
||||
pub(super) fn fractions_of(
|
||||
bounds: (u32, u32, u32, u32),
|
||||
dimensions: Option<(u32, u32)>,
|
||||
) -> [f64; 4] {
|
||||
let (width, height) = dimensions.unwrap_or((1, 1));
|
||||
let (w, h) = (f64::from(width), f64::from(height));
|
||||
[
|
||||
f64::from(bounds.0) / w,
|
||||
f64::from(bounds.1) / h,
|
||||
f64::from(bounds.2 - bounds.0 + 1) / w,
|
||||
f64::from(bounds.3 - bounds.1 + 1) / h,
|
||||
]
|
||||
}
|
||||
|
||||
/// Fractions of a display's frame `[x, y, width, height]` in global points.
|
||||
pub(super) fn points_of(frame_box: Option<[f64; 4]>, bbox: [f64; 4]) -> Option<[f64; 4]> {
|
||||
frame_box.map(|[x, y, frame_width, frame_height]| {
|
||||
[
|
||||
x + bbox[0] * frame_width,
|
||||
y + bbox[1] * frame_height,
|
||||
bbox[2] * frame_width,
|
||||
bbox[3] * frame_height,
|
||||
]
|
||||
})
|
||||
}
|
||||
|
||||
impl Analysis {
|
||||
@@ -247,86 +395,22 @@ impl Analysis {
|
||||
.find(|entry| entry.result.display.id == *display)
|
||||
.map(|entry| entry.result.display.frame);
|
||||
|
||||
let mut previous: Option<(Vec<u8>, u32, u32)> = None;
|
||||
let mut open: Option<Open> = None;
|
||||
let mut finished: Vec<Open> = Vec::new();
|
||||
|
||||
examined += video.scan(
|
||||
let tracked = track(
|
||||
video,
|
||||
*display,
|
||||
from_s,
|
||||
to_s,
|
||||
Scale::Width(COMPARE_WIDTH),
|
||||
&mut |decoded| {
|
||||
if cancel
|
||||
.is_some_and(|cancel| cancel.load(std::sync::atomic::Ordering::Relaxed))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
let now = luma(decoded);
|
||||
let (width, height) = (decoded.frame.width, decoded.frame.height);
|
||||
|
||||
if let Some((before, before_width, before_height)) = &previous
|
||||
&& (*before_width, *before_height) == (width, height)
|
||||
{
|
||||
let (mut count, mut min_x, mut min_y, mut max_x, mut max_y) =
|
||||
(0_usize, u32::MAX, u32::MAX, 0_u32, 0_u32);
|
||||
for (index, (a, b)) in before.iter().zip(&now).enumerate() {
|
||||
if (i32::from(*a) - i32::from(*b)).abs() > PIXEL_THRESHOLD {
|
||||
let (x, y) = (index as u32 % width, index as u32 / width);
|
||||
count += 1;
|
||||
min_x = min_x.min(x);
|
||||
max_x = max_x.max(x);
|
||||
min_y = min_y.min(y);
|
||||
max_y = max_y.max(y);
|
||||
}
|
||||
}
|
||||
|
||||
let share = count as f64 / now.len().max(1) as f64;
|
||||
if share >= threshold {
|
||||
match &mut open {
|
||||
Some(span) if decoded.shown_s - span.end <= MERGE_GAP_S => {
|
||||
span.end = decoded.shown_s;
|
||||
span.frames += 1;
|
||||
span.peak = span.peak.max(share);
|
||||
span.bounds = (
|
||||
span.bounds.0.min(min_x),
|
||||
span.bounds.1.min(min_y),
|
||||
span.bounds.2.max(max_x),
|
||||
span.bounds.3.max(max_y),
|
||||
);
|
||||
}
|
||||
_ => {
|
||||
finished.extend(open.take());
|
||||
open = Some(Open {
|
||||
start: decoded.shown_s,
|
||||
end: decoded.shown_s,
|
||||
frames: 1,
|
||||
peak: share,
|
||||
bounds: (min_x, min_y, max_x, max_y),
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
previous = Some((now, width, height));
|
||||
true
|
||||
&Tracking {
|
||||
threshold,
|
||||
crop: None,
|
||||
scale: Scale::Width(COMPARE_WIDTH),
|
||||
cancel,
|
||||
},
|
||||
)?;
|
||||
finished.extend(open.take());
|
||||
examined += tracked.examined;
|
||||
|
||||
let dimensions = previous
|
||||
.as_ref()
|
||||
.map(|(_, width, height)| (*width, *height));
|
||||
for span in finished {
|
||||
let (width, height) = dimensions.unwrap_or((1, 1));
|
||||
let (w, h) = (f64::from(width), f64::from(height));
|
||||
let bbox = [
|
||||
f64::from(span.bounds.0) / w,
|
||||
f64::from(span.bounds.1) / h,
|
||||
f64::from(span.bounds.2 - span.bounds.0 + 1) / w,
|
||||
f64::from(span.bounds.3 - span.bounds.1 + 1) / h,
|
||||
];
|
||||
for span in tracked.spans {
|
||||
let bbox = fractions_of(span.bounds, tracked.dimensions);
|
||||
let events = self
|
||||
.events()
|
||||
.iter()
|
||||
@@ -344,14 +428,7 @@ impl Analysis {
|
||||
frames: span.frames,
|
||||
peak_share: span.peak,
|
||||
bbox,
|
||||
bbox_points: frame_box.map(|[x, y, frame_width, frame_height]| {
|
||||
[
|
||||
x + bbox[0] * frame_width,
|
||||
y + bbox[1] * frame_height,
|
||||
bbox[2] * frame_width,
|
||||
bbox[3] * frame_height,
|
||||
]
|
||||
}),
|
||||
bbox_points: points_of(frame_box, bbox),
|
||||
events,
|
||||
});
|
||||
}
|
||||
@@ -442,3 +519,37 @@ impl Analysis {
|
||||
gaps
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn frames_close_together_are_one_stretch_and_a_pause_starts_another() {
|
||||
let mut segments = Segments::default();
|
||||
segments.push(1.0, 0.10, 5.0, (4, 4, 5, 5));
|
||||
segments.push(1.1, 0.20, 9.0, (2, 6, 3, 8)); // 0.1 s later: the same stretch
|
||||
segments.push(2.0, 0.05, 1.0, (0, 0, 0, 0)); // a second later: another
|
||||
|
||||
let found = segments.finish();
|
||||
assert_eq!(found.len(), 2);
|
||||
assert_eq!(
|
||||
(found[0].start, found[0].end, found[0].frames),
|
||||
(1.0, 1.1, 2)
|
||||
);
|
||||
assert_eq!((found[0].peak, found[0].peak_delta), (0.20, 9.0));
|
||||
assert_eq!(found[0].bounds, (2, 4, 5, 8));
|
||||
assert_eq!((found[1].start, found[1].frames), (2.0, 1));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_box_is_fractions_of_the_frame_and_then_points_of_the_display() {
|
||||
let bbox = fractions_of((10, 5, 19, 9), Some((100, 50)));
|
||||
assert_eq!(bbox, [0.1, 0.1, 0.1, 0.1]);
|
||||
assert_eq!(
|
||||
points_of(Some([1000.0, 0.0, 2000.0, 1000.0]), bbox),
|
||||
Some([1200.0, 100.0, 200.0, 100.0])
|
||||
);
|
||||
assert_eq!(points_of(None, bbox), None);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,520 @@
|
||||
//! `compare`: two recordings side by side, as numbers.
|
||||
//!
|
||||
//! For a before-and-after of a fix, or a good run against a bad one: how long each recording is,
|
||||
//! how many frames it has and how steadily they came, how much the screen moved, what the analysis
|
||||
//! found wrong, and how many annotations it carries, with the difference (`b` minus `a`) of each.
|
||||
//!
|
||||
//! | Measure | Where it comes from |
|
||||
//! |---|---|
|
||||
//! | `duration_s`, `frames`, `effective_fps` | the `frames-*.csv` logs of every display segment: real frames, and the intervals between them over the time they span |
|
||||
//! | `median_gap_ms`, `worst_gap_ms`, `late_gaps` | the gaps between consecutive real frames of a segment; `late_gaps` counts those over 100 ms and over twice the median |
|
||||
//! | `motion` | [`Analysis::motion`] over every display at sensitivity 1: segments, their total time, the largest share and mean brightness delta |
|
||||
//! | `findings` | [`Analysis::findings`], counted by kind |
|
||||
//! | `annotations`, `bugs` | the session's annotations |
|
||||
//!
|
||||
//! Frames come when the screen changes, so a still screen has long gaps and a low effective
|
||||
//! frame rate without anything being wrong: compare like with like.
|
||||
|
||||
use super::Analysis;
|
||||
use super::motion::ScanOptions;
|
||||
use super::video::SessionVideo;
|
||||
use crate::annotations;
|
||||
use color_eyre::eyre;
|
||||
use schemars::JsonSchema;
|
||||
use serde::Serialize;
|
||||
use std::collections::BTreeMap;
|
||||
use std::sync::atomic::AtomicBool;
|
||||
|
||||
/// The schema name that identifies a [`CompareData`] in an envelope (`compare/1`).
|
||||
pub const SCHEMA_COMPARE: &str = "compare/1";
|
||||
|
||||
/// A gap between frames over this is late (and over twice the median).
|
||||
const LATE_GAP_S: f64 = 0.1;
|
||||
|
||||
/// How the pictures of a recording moved.
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, JsonSchema)]
|
||||
pub struct MotionMetrics {
|
||||
/// How many segments of motion, over all displays.
|
||||
pub segments: usize,
|
||||
/// Their total duration, in seconds.
|
||||
pub active_s: f64,
|
||||
/// The largest share of a display that changed between two frames.
|
||||
pub peak_share: f64,
|
||||
/// The largest mean brightness difference between two frames, 0 to 255.
|
||||
pub peak_mean_delta: f64,
|
||||
}
|
||||
|
||||
/// The measures of one recording.
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, JsonSchema)]
|
||||
pub struct Side {
|
||||
/// The session directory.
|
||||
pub session: String,
|
||||
/// The length of the video, in seconds.
|
||||
pub duration_s: f64,
|
||||
/// Real frames over every display segment.
|
||||
pub frames: usize,
|
||||
/// Intervals between frames over the time they span, per second; `0` with fewer than two
|
||||
/// frames.
|
||||
pub effective_fps: f64,
|
||||
/// The median gap between consecutive frames, in milliseconds.
|
||||
pub median_gap_ms: f64,
|
||||
/// The longest gap between consecutive frames, in milliseconds.
|
||||
pub worst_gap_ms: f64,
|
||||
/// When the longest gap began, in seconds of video time.
|
||||
pub worst_gap_at_video_s: Option<f64>,
|
||||
/// Gaps over 100 ms and over twice the median.
|
||||
pub late_gaps: usize,
|
||||
/// The motion measures; `None` if the video could not be decoded (see `notes`).
|
||||
pub motion: Option<MotionMetrics>,
|
||||
/// How many findings of each kind.
|
||||
pub findings: BTreeMap<String, usize>,
|
||||
/// How many findings in all.
|
||||
pub findings_total: usize,
|
||||
/// How many annotations of any kind.
|
||||
pub annotations: usize,
|
||||
/// How many of them are bug regions.
|
||||
pub bugs: usize,
|
||||
}
|
||||
|
||||
/// `b` minus `a` for each measure. A motion measure is `None` if either side has none.
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, JsonSchema)]
|
||||
pub struct Deltas {
|
||||
/// Duration, in seconds.
|
||||
pub duration_s: f64,
|
||||
/// Frames.
|
||||
pub frames: i64,
|
||||
/// Effective frames per second.
|
||||
pub effective_fps: f64,
|
||||
/// Median frame gap, in milliseconds.
|
||||
pub median_gap_ms: f64,
|
||||
/// Worst frame gap, in milliseconds.
|
||||
pub worst_gap_ms: f64,
|
||||
/// Late gaps.
|
||||
pub late_gaps: i64,
|
||||
/// Motion segments.
|
||||
pub motion_segments: Option<i64>,
|
||||
/// Total time in motion, in seconds.
|
||||
pub active_s: Option<f64>,
|
||||
/// Peak share of a display changed.
|
||||
pub peak_share: Option<f64>,
|
||||
/// Peak mean brightness delta.
|
||||
pub peak_mean_delta: Option<f64>,
|
||||
/// Findings in all.
|
||||
pub findings_total: i64,
|
||||
/// Findings by kind, for every kind either side has.
|
||||
pub findings: BTreeMap<String, i64>,
|
||||
/// Annotations.
|
||||
pub annotations: i64,
|
||||
}
|
||||
|
||||
/// Two recordings side by side: the `data` of `compare`.
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, JsonSchema)]
|
||||
pub struct CompareData {
|
||||
/// The first recording.
|
||||
pub a: Side,
|
||||
/// The second recording.
|
||||
pub b: Side,
|
||||
/// `b` minus `a`.
|
||||
pub deltas: Deltas,
|
||||
/// What could not be measured, and why.
|
||||
pub notes: Vec<String>,
|
||||
}
|
||||
|
||||
/// The gaps between real frames in the text of a `frames-*.csv`, as `(time of the earlier frame
|
||||
/// in ns, gap in ns)`, and how many frames there were. Rows that do not parse (a header, a torn
|
||||
/// last row) and repeat rows (`-`) are ignored.
|
||||
///
|
||||
/// # Example
|
||||
///
|
||||
/// ```rust
|
||||
/// use komorebi_recorder::analysis::compare::frame_gaps;
|
||||
///
|
||||
/// let (gaps, frames) = frame_gaps("index,pts,received\n0,1000,1\n-,1500,1\n1,3000,1\n2,3500,1\n3,");
|
||||
/// assert_eq!(frames, 3);
|
||||
/// assert_eq!(gaps, [(1000, 2000), (3000, 500)]);
|
||||
/// ```
|
||||
#[must_use]
|
||||
pub fn frame_gaps(csv: &str) -> (Vec<(u64, u64)>, usize) {
|
||||
let mut gaps = Vec::new();
|
||||
let mut frames = 0;
|
||||
let mut previous: Option<u64> = None;
|
||||
for line in csv.lines() {
|
||||
let mut fields = line.split(',');
|
||||
let (Some(index), Some(pts), Some(_), None) =
|
||||
(fields.next(), fields.next(), fields.next(), fields.next())
|
||||
else {
|
||||
continue;
|
||||
};
|
||||
if index.parse::<u64>().is_err() {
|
||||
continue;
|
||||
}
|
||||
let Ok(pts) = pts.parse::<u64>() else {
|
||||
continue;
|
||||
};
|
||||
|
||||
frames += 1;
|
||||
if let Some(before) = previous {
|
||||
gaps.push((before, pts.saturating_sub(before)));
|
||||
}
|
||||
previous = Some(pts);
|
||||
}
|
||||
(gaps, frames)
|
||||
}
|
||||
|
||||
impl Analysis {
|
||||
/// Measures this recording as one [`Side`] of a comparison, as the [module](self) describes.
|
||||
///
|
||||
/// The frame logs are read from the session. If the video cannot be decoded (the host has no
|
||||
/// decoder, or there is no video) the motion measures are `None` and the reason goes in
|
||||
/// `notes`. `cancel` stops the motion scan early.
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// Returns an error if the findings cannot be computed (a recorded state is missing or
|
||||
/// unreadable) or the annotations cannot be read.
|
||||
pub fn side(&self, cancel: Option<&AtomicBool>, notes: &mut Vec<String>) -> eyre::Result<Side> {
|
||||
let mut all_gaps: Vec<(u64, u64)> = Vec::new();
|
||||
let (mut frames, mut span_ns, mut intervals) = (0_usize, 0_u128, 0_usize);
|
||||
for entry in &self.manifest.displays {
|
||||
let path = self.session.path(&entry.result.frames_file);
|
||||
let Ok(text) = std::fs::read_to_string(&path) else {
|
||||
continue;
|
||||
};
|
||||
let (gaps, count) = frame_gaps(&text);
|
||||
frames += count;
|
||||
intervals += gaps.len();
|
||||
span_ns += gaps.iter().map(|(_, gap)| u128::from(*gap)).sum::<u128>();
|
||||
all_gaps.extend(gaps);
|
||||
}
|
||||
|
||||
let mut sorted: Vec<u64> = all_gaps.iter().map(|(_, gap)| *gap).collect();
|
||||
sorted.sort_unstable();
|
||||
let median_ns = sorted.get(sorted.len() / 2).copied().unwrap_or(0);
|
||||
let worst = all_gaps.iter().max_by_key(|(_, gap)| *gap).copied();
|
||||
let late_ns = (LATE_GAP_S * 1e9) as u64;
|
||||
let late_gaps = all_gaps
|
||||
.iter()
|
||||
.filter(|(_, gap)| *gap > late_ns && *gap > median_ns.saturating_mul(2))
|
||||
.count();
|
||||
|
||||
let video = SessionVideo::new(&self.manifest, self.guard());
|
||||
let displays = video.displays();
|
||||
let (_, duration) = self.manifest.video_span();
|
||||
let motion = if displays.is_empty() {
|
||||
notes.push(format!(
|
||||
"{} recorded no video, so there are no motion measures",
|
||||
self.session.dir.display()
|
||||
));
|
||||
None
|
||||
} else {
|
||||
match self.motion(
|
||||
&video,
|
||||
&ScanOptions {
|
||||
displays: &displays,
|
||||
from_s: 0.0,
|
||||
to_s: duration,
|
||||
crop: None,
|
||||
cancel,
|
||||
},
|
||||
1.0,
|
||||
) {
|
||||
Ok(data) => Some(MotionMetrics {
|
||||
segments: data.segments.len(),
|
||||
active_s: data.active_s(),
|
||||
peak_share: data
|
||||
.segments
|
||||
.iter()
|
||||
.map(|segment| segment.peak_share)
|
||||
.fold(0.0, f64::max),
|
||||
peak_mean_delta: data
|
||||
.segments
|
||||
.iter()
|
||||
.map(|segment| segment.peak_mean_delta)
|
||||
.fold(0.0, f64::max),
|
||||
}),
|
||||
Err(error) => {
|
||||
notes.push(format!(
|
||||
"{}: motion could not be measured: {error}",
|
||||
self.session.dir.display()
|
||||
));
|
||||
None
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
let mut findings: BTreeMap<String, usize> = BTreeMap::new();
|
||||
let found = self.findings()?;
|
||||
for finding in &found {
|
||||
*findings.entry(finding.kind.clone()).or_default() += 1;
|
||||
}
|
||||
let loaded = annotations::load(&self.session.dir)?;
|
||||
|
||||
Ok(Side {
|
||||
session: self.session.dir.display().to_string(),
|
||||
duration_s: duration,
|
||||
frames,
|
||||
effective_fps: if span_ns == 0 {
|
||||
0.0
|
||||
} else {
|
||||
intervals as f64 / (span_ns as f64 / 1e9)
|
||||
},
|
||||
median_gap_ms: median_ns as f64 / 1e6,
|
||||
worst_gap_ms: worst.map_or(0.0, |(_, gap)| gap as f64 / 1e6),
|
||||
worst_gap_at_video_s: worst.map(|(at, _)| self.video_s(at)),
|
||||
late_gaps,
|
||||
motion,
|
||||
findings_total: found.len(),
|
||||
findings,
|
||||
annotations: loaded.annotations.len(),
|
||||
bugs: loaded.bugs().count(),
|
||||
})
|
||||
}
|
||||
|
||||
/// Compares this recording (`a`) with `other` (`b`): see the [module](self) documentation.
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// The same as [`Analysis::side`], for either recording.
|
||||
///
|
||||
/// # Example
|
||||
///
|
||||
/// A session compared with itself differs in nothing:
|
||||
///
|
||||
/// ```rust
|
||||
/// use komorebi_recorder::analysis::Analysis;
|
||||
/// use komorebi_recorder::testkit::SessionBuilder;
|
||||
///
|
||||
/// let (session, _) = SessionBuilder::new("doc-compare").length(5.0).build();
|
||||
/// let analysis = Analysis::open(&session.dir)?;
|
||||
/// let data = analysis.compare(&analysis, None)?;
|
||||
/// assert_eq!(data.a.duration_s, data.b.duration_s);
|
||||
/// assert_eq!(data.deltas.duration_s, 0.0);
|
||||
/// assert_eq!(data.deltas.findings_total, 0);
|
||||
/// # Ok::<(), Box<dyn std::error::Error>>(())
|
||||
/// ```
|
||||
pub fn compare(
|
||||
&self,
|
||||
other: &Analysis,
|
||||
cancel: Option<&AtomicBool>,
|
||||
) -> eyre::Result<CompareData> {
|
||||
let mut notes = Vec::new();
|
||||
let a = self.side(cancel, &mut notes)?;
|
||||
let b = other.side(cancel, &mut notes)?;
|
||||
let deltas = deltas(&a, &b);
|
||||
Ok(CompareData {
|
||||
a,
|
||||
b,
|
||||
deltas,
|
||||
notes,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
fn deltas(a: &Side, b: &Side) -> Deltas {
|
||||
let both = |f: fn(&MotionMetrics) -> f64| match (&a.motion, &b.motion) {
|
||||
(Some(x), Some(y)) => Some(f(y) - f(x)),
|
||||
_ => None,
|
||||
};
|
||||
let kinds: std::collections::BTreeSet<&String> =
|
||||
a.findings.keys().chain(b.findings.keys()).collect();
|
||||
Deltas {
|
||||
duration_s: b.duration_s - a.duration_s,
|
||||
frames: b.frames as i64 - a.frames as i64,
|
||||
effective_fps: b.effective_fps - a.effective_fps,
|
||||
median_gap_ms: b.median_gap_ms - a.median_gap_ms,
|
||||
worst_gap_ms: b.worst_gap_ms - a.worst_gap_ms,
|
||||
late_gaps: b.late_gaps as i64 - a.late_gaps as i64,
|
||||
motion_segments: both(|m| m.segments as f64).map(|d| d as i64),
|
||||
active_s: both(|m| m.active_s),
|
||||
peak_share: both(|m| m.peak_share),
|
||||
peak_mean_delta: both(|m| m.peak_mean_delta),
|
||||
findings_total: b.findings_total as i64 - a.findings_total as i64,
|
||||
findings: kinds
|
||||
.into_iter()
|
||||
.map(|kind| {
|
||||
let count = |side: &Side| side.findings.get(kind).copied().unwrap_or(0) as i64;
|
||||
(kind.clone(), count(b) - count(a))
|
||||
})
|
||||
.collect(),
|
||||
annotations: b.annotations as i64 - a.annotations as i64,
|
||||
}
|
||||
}
|
||||
|
||||
impl CompareData {
|
||||
/// Renders the comparison as a table with a column for each recording and one for the
|
||||
/// difference.
|
||||
///
|
||||
/// # Example
|
||||
///
|
||||
/// ```rust
|
||||
/// use komorebi_recorder::analysis::Analysis;
|
||||
/// use komorebi_recorder::testkit::SessionBuilder;
|
||||
///
|
||||
/// let (session, _) = SessionBuilder::new("doc-compare-text").length(5.0).build();
|
||||
/// let analysis = Analysis::open(&session.dir)?;
|
||||
/// let text = analysis.compare(&analysis, None)?.to_text();
|
||||
/// assert!(text.contains("duration"));
|
||||
/// assert!(text.contains("findings"));
|
||||
/// # Ok::<(), Box<dyn std::error::Error>>(())
|
||||
/// ```
|
||||
#[must_use]
|
||||
pub fn to_text(&self) -> String {
|
||||
use std::fmt::Write;
|
||||
|
||||
let (a, b, d) = (&self.a, &self.b, &self.deltas);
|
||||
let mut out = String::new();
|
||||
let _ = writeln!(out, "a: {}\nb: {}\n", a.session, b.session);
|
||||
let _ = writeln!(out, "{:<24}{:>14}{:>14}{:>14}", "", "a", "b", "b - a");
|
||||
let mut row = |name: &str, x: String, y: String, delta: String| {
|
||||
let _ = writeln!(out, "{name:<24}{x:>14}{y:>14}{delta:>14}");
|
||||
};
|
||||
let number = |value: f64, places: usize| format!("{value:.places$}");
|
||||
let signed = |value: f64, places: usize| format!("{value:+.places$}");
|
||||
let whole = |value: i64| format!("{value:+}");
|
||||
|
||||
row(
|
||||
"duration (s)",
|
||||
number(a.duration_s, 2),
|
||||
number(b.duration_s, 2),
|
||||
signed(d.duration_s, 2),
|
||||
);
|
||||
row(
|
||||
"frames",
|
||||
a.frames.to_string(),
|
||||
b.frames.to_string(),
|
||||
whole(d.frames),
|
||||
);
|
||||
row(
|
||||
"effective fps",
|
||||
number(a.effective_fps, 1),
|
||||
number(b.effective_fps, 1),
|
||||
signed(d.effective_fps, 1),
|
||||
);
|
||||
row(
|
||||
"median frame gap (ms)",
|
||||
number(a.median_gap_ms, 1),
|
||||
number(b.median_gap_ms, 1),
|
||||
signed(d.median_gap_ms, 1),
|
||||
);
|
||||
row(
|
||||
"worst frame gap (ms)",
|
||||
number(a.worst_gap_ms, 1),
|
||||
number(b.worst_gap_ms, 1),
|
||||
signed(d.worst_gap_ms, 1),
|
||||
);
|
||||
row(
|
||||
"late frame gaps",
|
||||
a.late_gaps.to_string(),
|
||||
b.late_gaps.to_string(),
|
||||
whole(d.late_gaps),
|
||||
);
|
||||
|
||||
let dash = || "-".to_string();
|
||||
row(
|
||||
"motion segments",
|
||||
a.motion
|
||||
.as_ref()
|
||||
.map_or_else(dash, |m| m.segments.to_string()),
|
||||
b.motion
|
||||
.as_ref()
|
||||
.map_or_else(dash, |m| m.segments.to_string()),
|
||||
d.motion_segments.map_or_else(dash, whole),
|
||||
);
|
||||
row(
|
||||
"time in motion (s)",
|
||||
a.motion
|
||||
.as_ref()
|
||||
.map_or_else(dash, |m| number(m.active_s, 2)),
|
||||
b.motion
|
||||
.as_ref()
|
||||
.map_or_else(dash, |m| number(m.active_s, 2)),
|
||||
d.active_s.map_or_else(dash, |v| signed(v, 2)),
|
||||
);
|
||||
row(
|
||||
"peak motion (% pixels)",
|
||||
a.motion
|
||||
.as_ref()
|
||||
.map_or_else(dash, |m| number(m.peak_share * 100.0, 1)),
|
||||
b.motion
|
||||
.as_ref()
|
||||
.map_or_else(dash, |m| number(m.peak_share * 100.0, 1)),
|
||||
d.peak_share.map_or_else(dash, |v| signed(v * 100.0, 1)),
|
||||
);
|
||||
row(
|
||||
"peak mean delta",
|
||||
a.motion
|
||||
.as_ref()
|
||||
.map_or_else(dash, |m| number(m.peak_mean_delta, 1)),
|
||||
b.motion
|
||||
.as_ref()
|
||||
.map_or_else(dash, |m| number(m.peak_mean_delta, 1)),
|
||||
d.peak_mean_delta.map_or_else(dash, |v| signed(v, 1)),
|
||||
);
|
||||
row(
|
||||
"findings",
|
||||
a.findings_total.to_string(),
|
||||
b.findings_total.to_string(),
|
||||
whole(d.findings_total),
|
||||
);
|
||||
for (kind, delta) in &d.findings {
|
||||
row(
|
||||
&format!(" {kind}"),
|
||||
a.findings.get(kind).copied().unwrap_or(0).to_string(),
|
||||
b.findings.get(kind).copied().unwrap_or(0).to_string(),
|
||||
whole(*delta),
|
||||
);
|
||||
}
|
||||
row(
|
||||
"annotations",
|
||||
a.annotations.to_string(),
|
||||
b.annotations.to_string(),
|
||||
whole(d.annotations),
|
||||
);
|
||||
for note in &self.notes {
|
||||
let _ = writeln!(out, "note: {note}");
|
||||
}
|
||||
out
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn side(frames: usize, kinds: &[(&str, usize)]) -> Side {
|
||||
Side {
|
||||
session: String::new(),
|
||||
duration_s: 10.0,
|
||||
frames,
|
||||
effective_fps: 30.0,
|
||||
median_gap_ms: 33.0,
|
||||
worst_gap_ms: 100.0,
|
||||
worst_gap_at_video_s: None,
|
||||
late_gaps: 0,
|
||||
motion: None,
|
||||
findings: kinds.iter().map(|(k, n)| ((*k).to_string(), *n)).collect(),
|
||||
findings_total: kinds.iter().map(|(_, n)| n).sum(),
|
||||
annotations: 0,
|
||||
bugs: 0,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn frame_gaps_ignore_headers_repeats_and_torn_rows() {
|
||||
let (gaps, frames) = frame_gaps("0,100,100\n-,150,150\n1,300,300\n2,3");
|
||||
assert_eq!(frames, 2);
|
||||
assert_eq!(gaps, [(100, 200)]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn deltas_cover_the_kinds_of_either_side() {
|
||||
let a = side(100, &[("slow-handler", 2)]);
|
||||
let b = side(80, &[("event-storm", 1), ("slow-handler", 1)]);
|
||||
let d = deltas(&a, &b);
|
||||
assert_eq!(d.frames, -20);
|
||||
assert_eq!(d.findings_total, 0);
|
||||
assert_eq!(d.findings["slow-handler"], -1);
|
||||
assert_eq!(d.findings["event-storm"], 1);
|
||||
assert_eq!(d.motion_segments, None);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,339 @@
|
||||
//! `framediff`: how two frames differ, in numbers and as a picture.
|
||||
//!
|
||||
//! Two frames are decoded (the ones on screen at two moments of one recording, or one moment of
|
||||
//! each of two recordings), optionally cut to a region of the display, reduced to brightness and
|
||||
//! compared pixel by pixel. The answer is how much changed (the share of pixels that moved by more
|
||||
//! than 24 of 255, and the mean and largest brightness difference), the box around the change, and,
|
||||
//! if asked, a PNG of the compared area in which the changed pixels are red.
|
||||
//!
|
||||
//! Frames are decoded at a width of 1280 pixels unless asked otherwise (never more than the
|
||||
//! video has), so a region is compared at that resolution. Two recordings must have been made at
|
||||
//! the same size to be compared.
|
||||
|
||||
use super::Analysis;
|
||||
use super::envelope;
|
||||
use super::pixels;
|
||||
use super::pixels::Crop;
|
||||
use super::sheet::Canvas;
|
||||
use super::video::SessionVideo;
|
||||
use crate::av::Scale;
|
||||
use color_eyre::eyre;
|
||||
use schemars::JsonSchema;
|
||||
use serde::Serialize;
|
||||
use std::path::Path;
|
||||
|
||||
/// The schema name that identifies a [`FrameDiffData`] in an envelope (`framediff/1`).
|
||||
pub const SCHEMA_FRAMEDIFF: &str = "framediff/1";
|
||||
|
||||
/// The width frames are decoded at by default.
|
||||
pub const DEFAULT_WIDTH: u32 = 1280;
|
||||
|
||||
/// One frame to compare: a moment on a display of a recording.
|
||||
#[derive(Clone, Copy)]
|
||||
pub struct FrameAt<'a> {
|
||||
/// The recording.
|
||||
pub analysis: &'a Analysis,
|
||||
/// Its video, made on this thread.
|
||||
pub video: &'a SessionVideo<'a>,
|
||||
/// The display.
|
||||
pub display: u32,
|
||||
/// The moment, in seconds of video time of that recording.
|
||||
pub time_s: f64,
|
||||
}
|
||||
|
||||
/// What to compare and what to write.
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
pub struct FrameDiffOptions<'a> {
|
||||
/// Only this part of the display, in global display points (of the first frame's display);
|
||||
/// the whole display if `None`.
|
||||
pub crop: Option<Crop>,
|
||||
/// The width to decode frames at.
|
||||
pub width: u32,
|
||||
/// Where to write the PNG of the change, if wanted.
|
||||
pub png: Option<&'a Path>,
|
||||
}
|
||||
|
||||
/// How two frames differ: the `data` of `framediff`.
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, JsonSchema)]
|
||||
pub struct FrameDiffData {
|
||||
/// The display of the first frame.
|
||||
pub display: u32,
|
||||
/// The display of the second frame (differs only when `against` was used).
|
||||
pub display_b: u32,
|
||||
/// The moment asked for the first frame, in seconds of video time.
|
||||
pub time_a_s: f64,
|
||||
/// When the frame that was on screen then was shown.
|
||||
pub shown_a_s: f64,
|
||||
/// The moment asked for the second frame, in seconds of video time.
|
||||
pub time_b_s: f64,
|
||||
/// When the frame that was on screen then was shown.
|
||||
pub shown_b_s: f64,
|
||||
/// The other recording, if the second frame is from one.
|
||||
pub against: Option<String>,
|
||||
/// The region compared, in global display points; `None` is the whole display.
|
||||
pub region: Option<Crop>,
|
||||
/// The width of the compared area, in pixels.
|
||||
pub width: u32,
|
||||
/// The height of the compared area, in pixels.
|
||||
pub height: u32,
|
||||
/// How many pixels moved by more than 24 of 255 in brightness.
|
||||
pub changed_pixels: usize,
|
||||
/// The share of the compared pixels that changed, 0 to 1.
|
||||
pub changed_share: f64,
|
||||
/// The mean absolute brightness difference over the compared pixels, 0 to 255.
|
||||
pub mean_delta: f64,
|
||||
/// The largest brightness difference of any pixel, 0 to 255.
|
||||
pub peak_delta: u8,
|
||||
/// The box around the changed pixels as fractions of the display: `[x, y, width, height]`.
|
||||
/// `None` if nothing changed.
|
||||
pub bbox: Option<[f64; 4]>,
|
||||
/// The same box in global display points.
|
||||
pub bbox_points: Option<[f64; 4]>,
|
||||
/// Where the PNG was written, if one was asked for.
|
||||
pub png: Option<String>,
|
||||
}
|
||||
|
||||
impl FrameDiffData {
|
||||
/// Renders the answer as text.
|
||||
///
|
||||
/// # Example
|
||||
///
|
||||
/// ```rust
|
||||
/// use komorebi_recorder::analysis::framediff::FrameDiffData;
|
||||
///
|
||||
/// let data = FrameDiffData {
|
||||
/// display: 1, display_b: 1, time_a_s: 3.2, shown_a_s: 3.18, time_b_s: 3.5, shown_b_s: 3.47,
|
||||
/// against: None, region: None, width: 1280, height: 540, changed_pixels: 6912,
|
||||
/// changed_share: 0.01, mean_delta: 2.5, peak_delta: 200, bbox: Some([0.1, 0.1, 0.5, 0.5]),
|
||||
/// bbox_points: None, png: None,
|
||||
/// };
|
||||
/// assert!(data.to_text().contains("1.00% of the pixels changed"));
|
||||
/// ```
|
||||
#[must_use]
|
||||
pub fn to_text(&self) -> String {
|
||||
use crate::timecode::format as time;
|
||||
use std::fmt::Write;
|
||||
|
||||
let mut out = String::new();
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"display {}: {} (frame shown {}) against {} (frame shown {}){}",
|
||||
self.display,
|
||||
time(self.time_a_s),
|
||||
time(self.shown_a_s),
|
||||
time(self.time_b_s),
|
||||
time(self.shown_b_s),
|
||||
self.against
|
||||
.as_ref()
|
||||
.map_or_else(String::new, |other| format!(" of {other}")),
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"{:.2}% of the pixels changed ({} of {}x{}), mean brightness delta {:.2}, peak {}",
|
||||
self.changed_share * 100.0,
|
||||
self.changed_pixels,
|
||||
self.width,
|
||||
self.height,
|
||||
self.mean_delta,
|
||||
self.peak_delta,
|
||||
);
|
||||
match (self.bbox, self.bbox_points) {
|
||||
(Some(bbox), points) => {
|
||||
let _ = write!(
|
||||
out,
|
||||
"box of change {:.0}%,{:.0}% {:.0}%x{:.0}% of the display",
|
||||
bbox[0] * 100.0,
|
||||
bbox[1] * 100.0,
|
||||
bbox[2] * 100.0,
|
||||
bbox[3] * 100.0,
|
||||
);
|
||||
if let Some(p) = points {
|
||||
let _ = write!(
|
||||
out,
|
||||
" ({:.0},{:.0} {:.0}x{:.0} points)",
|
||||
p[0], p[1], p[2], p[3]
|
||||
);
|
||||
}
|
||||
out.push('\n');
|
||||
}
|
||||
(None, _) => out.push_str("nothing changed\n"),
|
||||
}
|
||||
if let Some(path) = &self.png {
|
||||
let _ = writeln!(out, "wrote {path}");
|
||||
}
|
||||
out
|
||||
}
|
||||
}
|
||||
|
||||
fn grab(at: &FrameAt, scale: Scale, what: &str) -> eyre::Result<super::video::DecodedFrame> {
|
||||
at.video
|
||||
.frame_at(at.display, at.time_s, scale)?
|
||||
.ok_or_else(|| {
|
||||
envelope::not_found(format!(
|
||||
"display {} had no video at {} ({what})",
|
||||
at.display,
|
||||
crate::timecode::format(at.time_s)
|
||||
))
|
||||
})
|
||||
}
|
||||
|
||||
/// Compares the frame `a` with the frame `b`, as the [module](self) describes.
|
||||
///
|
||||
/// The region is placed on the display of `a` (so it is in the points of that recording) and
|
||||
/// applied as fractions to both.
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// Returns an error if:
|
||||
///
|
||||
/// - the region does not overlap the display, or its frame is unknown (the usage kind);
|
||||
/// - a display had no video at its moment (not found);
|
||||
/// - the two frames are not the same size, which happens when the recordings were made at
|
||||
/// different sizes (the unusable kind);
|
||||
/// - a video cannot be opened or decoded, the host has no decoder, or the PNG cannot be written.
|
||||
///
|
||||
/// # Example
|
||||
///
|
||||
/// This needs recorded video, so it is not run here; `tests/video.rs` runs it through the
|
||||
/// `framediff` command.
|
||||
///
|
||||
/// ```rust,no_run
|
||||
/// use komorebi_recorder::analysis::Analysis;
|
||||
/// use komorebi_recorder::analysis::framediff::{DEFAULT_WIDTH, FrameAt, FrameDiffOptions, framediff};
|
||||
/// use komorebi_recorder::analysis::video::SessionVideo;
|
||||
///
|
||||
/// let analysis = Analysis::open(std::path::Path::new("session-dir"))?;
|
||||
/// let video = SessionVideo::new(&analysis.manifest, analysis.guard());
|
||||
/// let at = |time_s| FrameAt { analysis: &analysis, video: &video, display: 1, time_s };
|
||||
/// let data = framediff(
|
||||
/// &at(3.2),
|
||||
/// &at(3.5),
|
||||
/// &FrameDiffOptions { crop: None, width: DEFAULT_WIDTH, png: None },
|
||||
/// )?;
|
||||
/// print!("{}", data.to_text());
|
||||
/// # Ok::<(), Box<dyn std::error::Error>>(())
|
||||
/// ```
|
||||
pub fn framediff(
|
||||
a: &FrameAt,
|
||||
b: &FrameAt,
|
||||
options: &FrameDiffOptions,
|
||||
) -> eyre::Result<FrameDiffData> {
|
||||
let fractions = options
|
||||
.crop
|
||||
.as_ref()
|
||||
.map(|crop| a.analysis.crop_fractions(a.display, crop))
|
||||
.transpose()?;
|
||||
let scale = Scale::Width(options.width.max(2));
|
||||
let (first, second) = (grab(a, scale, "first")?, grab(b, scale, "second")?);
|
||||
let (width, height) = (first.frame.width, first.frame.height);
|
||||
if (second.frame.width, second.frame.height) != (width, height) {
|
||||
return Err(envelope::unusable(format!(
|
||||
"the frames are not the same size ({width}x{height} against {}x{}): the recordings were made at different sizes",
|
||||
second.frame.width, second.frame.height
|
||||
)));
|
||||
}
|
||||
|
||||
let rect = fractions.map_or(
|
||||
pixels::PixelRect {
|
||||
x: 0,
|
||||
y: 0,
|
||||
width,
|
||||
height,
|
||||
},
|
||||
|fractions| pixels::pixel_rect(fractions, width, height),
|
||||
);
|
||||
let before = pixels::crop_luma(&pixels::luma(&first.frame.bgra), width, rect);
|
||||
let after = pixels::crop_luma(&pixels::luma(&second.frame.bgra), width, rect);
|
||||
let found = pixels::delta(&before, &after, rect.width, super::changes::PIXEL_THRESHOLD);
|
||||
|
||||
let bbox = found.bounds.map(|(min_x, min_y, max_x, max_y)| {
|
||||
super::changes::fractions_of(
|
||||
(
|
||||
min_x + rect.x,
|
||||
min_y + rect.y,
|
||||
max_x + rect.x,
|
||||
max_y + rect.y,
|
||||
),
|
||||
Some((width, height)),
|
||||
)
|
||||
});
|
||||
|
||||
if let Some(path) = options.png {
|
||||
write_png(
|
||||
path,
|
||||
&second.frame.bgra,
|
||||
width,
|
||||
rect,
|
||||
&before,
|
||||
&after,
|
||||
found.bounds,
|
||||
)?;
|
||||
}
|
||||
|
||||
Ok(FrameDiffData {
|
||||
display: a.display,
|
||||
display_b: b.display,
|
||||
time_a_s: a.time_s,
|
||||
shown_a_s: first.shown_s,
|
||||
time_b_s: b.time_s,
|
||||
shown_b_s: second.shown_s,
|
||||
against: (!std::ptr::eq(a.analysis, b.analysis))
|
||||
.then(|| b.analysis.session.dir.display().to_string()),
|
||||
region: options.crop,
|
||||
width: rect.width,
|
||||
height: rect.height,
|
||||
changed_pixels: found.changed,
|
||||
changed_share: found.share(),
|
||||
mean_delta: found.mean,
|
||||
peak_delta: found.peak,
|
||||
bbox,
|
||||
bbox_points: bbox
|
||||
.and_then(|bbox| super::changes::points_of(a.analysis.display_frame(a.display), bbox)),
|
||||
png: options.png.map(|path| path.display().to_string()),
|
||||
})
|
||||
}
|
||||
|
||||
/// Writes the compared area of the second frame, dimmed, with the pixels that changed in red and
|
||||
/// a yellow box around them.
|
||||
fn write_png(
|
||||
path: &Path,
|
||||
bgra: &[u8],
|
||||
width: u32,
|
||||
rect: pixels::PixelRect,
|
||||
before: &[u8],
|
||||
after: &[u8],
|
||||
bounds: Option<(u32, u32, u32, u32)>,
|
||||
) -> eyre::Result<()> {
|
||||
let area = pixels::crop_bgra(bgra, width, rect);
|
||||
let mut canvas = Canvas::new(rect.width, rect.height);
|
||||
for row in 0..rect.height {
|
||||
for column in 0..rect.width {
|
||||
let at = (row * rect.width + column) as usize;
|
||||
let changed = (i32::from(before[at]) - i32::from(after[at])).abs()
|
||||
> super::changes::PIXEL_THRESHOLD;
|
||||
let color = if changed {
|
||||
[255, 40, 40]
|
||||
} else {
|
||||
let grey = (u32::from(area[at * 4 + 2]) * 299
|
||||
+ u32::from(area[at * 4 + 1]) * 587
|
||||
+ u32::from(area[at * 4]) * 114)
|
||||
/ 1000;
|
||||
let dimmed = (grey * 2 / 5) as u8;
|
||||
[dimmed, dimmed, dimmed]
|
||||
};
|
||||
canvas.put(i64::from(column), i64::from(row), color, 1.0);
|
||||
}
|
||||
}
|
||||
if let Some((min_x, min_y, max_x, max_y)) = bounds {
|
||||
canvas.border(
|
||||
min_x,
|
||||
min_y,
|
||||
max_x - min_x + 1,
|
||||
max_y - min_y + 1,
|
||||
1,
|
||||
[255, 220, 0],
|
||||
);
|
||||
}
|
||||
|
||||
canvas.save(path)
|
||||
}
|
||||
@@ -8,6 +8,7 @@
|
||||
//! bound ([`budget`]).
|
||||
|
||||
pub mod answers;
|
||||
pub mod blank;
|
||||
pub mod blocks;
|
||||
pub mod brief;
|
||||
pub mod budget;
|
||||
@@ -15,11 +16,13 @@ pub mod catalog;
|
||||
pub mod changes;
|
||||
pub mod classify;
|
||||
pub mod code;
|
||||
pub mod compare;
|
||||
pub mod diff;
|
||||
pub mod envelope;
|
||||
pub mod events;
|
||||
pub mod facade;
|
||||
pub mod flavor;
|
||||
pub mod framediff;
|
||||
pub mod ghost;
|
||||
pub mod guard;
|
||||
pub mod history;
|
||||
@@ -28,8 +31,10 @@ pub mod invariants;
|
||||
pub mod logparse;
|
||||
pub mod logq;
|
||||
pub mod lookup;
|
||||
pub mod motion;
|
||||
pub mod os_trace;
|
||||
pub mod overview;
|
||||
pub mod pixels;
|
||||
pub mod probed;
|
||||
pub mod schemas;
|
||||
pub mod search;
|
||||
|
||||
@@ -0,0 +1,417 @@
|
||||
//! `motion`: when the picture of a display, or of a part of it, was moving.
|
||||
//!
|
||||
//! This is [`changes`](super::changes) with a region: the same frame-to-frame comparison, but it
|
||||
//! can be limited to a rectangle of a display ([`ScanOptions::crop`], in display points), and each
|
||||
//! segment of activity reports how hard the picture moved as well as how much of it did.
|
||||
//!
|
||||
//! A segment is a run of frames in which at least 0.05% (divided by the sensitivity) of the
|
||||
//! compared pixels changed by more than 24 of 255 brightness, with frames closer than 0.15 s
|
||||
//! belonging to one segment; see [`super::changes`] for the details. Per segment it reports:
|
||||
//!
|
||||
//! - `peak_share`: the largest share of the compared pixels that changed between two frames;
|
||||
//! - `peak_mean_delta`: the largest mean brightness difference (0 to 255) between two frames,
|
||||
//! which tells a faint flicker over the whole region from a few pixels flipping fully.
|
||||
//!
|
||||
//! Where a region is given the compared pixels are those of the region only, so `peak_share` is a
|
||||
//! share of the region. The box of change is still given as a fraction of the display and in
|
||||
//! display points, so it can be used with `--region` of the other commands.
|
||||
//!
|
||||
//! A region is cut from a frame decoded wide enough to give it at least 240 pixels across, so a
|
||||
//! small region costs more to scan than a whole display.
|
||||
|
||||
use super::Analysis;
|
||||
use super::changes::Gap;
|
||||
use super::changes::Tracking;
|
||||
use super::changes::fractions_of;
|
||||
use super::changes::points_of;
|
||||
use super::changes::track;
|
||||
use super::envelope;
|
||||
use super::pixels::Crop;
|
||||
use super::video::SessionVideo;
|
||||
use crate::av::Scale;
|
||||
use color_eyre::eyre;
|
||||
use schemars::JsonSchema;
|
||||
use serde::Serialize;
|
||||
use std::sync::atomic::AtomicBool;
|
||||
|
||||
/// The schema name that identifies a [`MotionData`] in an envelope (`motion/1`).
|
||||
pub const SCHEMA_MOTION: &str = "motion/1";
|
||||
|
||||
/// Whole displays are compared at this width, like `changes`.
|
||||
const WHOLE_WIDTH: u32 = 240;
|
||||
|
||||
/// A region is decoded wide enough to have this many pixels across.
|
||||
const REGION_PIXELS: f64 = 240.0;
|
||||
|
||||
/// The widest a frame is decoded for a region.
|
||||
const MAX_WIDTH: u32 = 3840;
|
||||
|
||||
/// What to scan: shared by `motion` and `blank`.
|
||||
///
|
||||
/// `crop` is where a window's rect goes, once a window track can say where a window was: it is in
|
||||
/// the same global display points as a `bbox_points`, so a [`Crop::from_rect`] of a window's rect
|
||||
/// is all it takes.
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
pub struct ScanOptions<'a> {
|
||||
/// The displays to scan; each must be a display of the session.
|
||||
pub displays: &'a [u32],
|
||||
/// The start of the range, in seconds of video time.
|
||||
pub from_s: f64,
|
||||
/// The end of the range, in seconds of video time.
|
||||
pub to_s: f64,
|
||||
/// Only this part of each display, in global display points; the whole display if `None`.
|
||||
pub crop: Option<Crop>,
|
||||
/// Set from another thread to stop early with what was found so far.
|
||||
pub cancel: Option<&'a AtomicBool>,
|
||||
}
|
||||
|
||||
/// A stretch of video in which the picture of a display, or of a region of it, was moving.
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, JsonSchema)]
|
||||
pub struct MotionSegment {
|
||||
/// The id of the display.
|
||||
pub display: u32,
|
||||
/// When the first changed frame was on screen, in seconds of video time.
|
||||
pub start_video_s: f64,
|
||||
/// When the last changed frame was on screen, in seconds of video time.
|
||||
pub end_video_s: f64,
|
||||
/// `end_video_s - start_video_s`; zero for a single changed frame.
|
||||
pub duration_s: f64,
|
||||
/// Changed frames in the segment.
|
||||
pub frames: usize,
|
||||
/// The largest share of the compared pixels (the region's, if there is one) that changed
|
||||
/// between two consecutive frames.
|
||||
pub peak_share: f64,
|
||||
/// The largest mean brightness difference, 0 to 255, over the compared pixels between two
|
||||
/// consecutive frames.
|
||||
pub peak_mean_delta: f64,
|
||||
/// The box around everything that changed, as fractions of the display: `[x, y, width, height]`.
|
||||
pub bbox: [f64; 4],
|
||||
/// The same box in global display points, if the display's frame is known.
|
||||
pub bbox_points: Option<[f64; 4]>,
|
||||
/// The `seq` of each komorebi event received within 0.25 s of the segment.
|
||||
pub events: Vec<u64>,
|
||||
}
|
||||
|
||||
/// Where a display, or a region of it, was moving over a range: the `data` of `motion`.
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, JsonSchema)]
|
||||
pub struct MotionData {
|
||||
/// The start of the range scanned, in seconds of video time.
|
||||
pub from_video_s: f64,
|
||||
/// The end of the range scanned, in seconds of video time.
|
||||
pub to_video_s: f64,
|
||||
/// The displays scanned.
|
||||
pub displays: Vec<u32>,
|
||||
/// The region compared, in global display points, if the scan was limited to one.
|
||||
pub region: Option<Crop>,
|
||||
/// The sensitivity asked for.
|
||||
pub sensitivity: f64,
|
||||
/// How many frames were decoded and looked at, over all displays. A cancelled scan counts
|
||||
/// only what it got through.
|
||||
pub frames_examined: usize,
|
||||
/// The segments of activity, ordered by start and then display.
|
||||
pub segments: Vec<MotionSegment>,
|
||||
/// Stretches of the range where a display had no video at all.
|
||||
pub gaps: Vec<Gap>,
|
||||
}
|
||||
|
||||
impl MotionData {
|
||||
/// The total time covered by segments, in seconds, counting each display's segments (they
|
||||
/// never overlap within a display) and adding displays together.
|
||||
#[must_use]
|
||||
pub fn active_s(&self) -> f64 {
|
||||
self.segments.iter().map(|segment| segment.duration_s).sum()
|
||||
}
|
||||
|
||||
/// Renders the answer as text, one line for each segment.
|
||||
///
|
||||
/// # Example
|
||||
///
|
||||
/// ```rust
|
||||
/// use komorebi_recorder::analysis::motion::{MotionData, MotionSegment};
|
||||
///
|
||||
/// let data = MotionData {
|
||||
/// from_video_s: 0.0,
|
||||
/// to_video_s: 6.0,
|
||||
/// displays: vec![1],
|
||||
/// region: None,
|
||||
/// sensitivity: 1.0,
|
||||
/// frames_examined: 100,
|
||||
/// segments: vec![MotionSegment {
|
||||
/// display: 1,
|
||||
/// start_video_s: 1.0,
|
||||
/// end_video_s: 1.5,
|
||||
/// duration_s: 0.5,
|
||||
/// frames: 10,
|
||||
/// peak_share: 0.25,
|
||||
/// peak_mean_delta: 12.5,
|
||||
/// bbox: [0.0, 0.0, 1.0, 1.0],
|
||||
/// bbox_points: None,
|
||||
/// events: vec![],
|
||||
/// }],
|
||||
/// gaps: vec![],
|
||||
/// };
|
||||
/// let text = data.to_text();
|
||||
/// assert!(text.starts_with("100 frames compared on display(s) 1"));
|
||||
/// assert!(text.contains("peak 25.0% of the pixels, mean delta 12.5"));
|
||||
/// ```
|
||||
#[must_use]
|
||||
pub fn to_text(&self) -> String {
|
||||
use crate::timecode::format as time;
|
||||
use std::fmt::Write;
|
||||
|
||||
let mut out = String::new();
|
||||
let region = self.region.map_or_else(String::new, |crop| {
|
||||
format!(
|
||||
" in region {},{} {}x{}",
|
||||
crop.x, crop.y, crop.width, crop.height
|
||||
)
|
||||
});
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"{} frames compared on display(s) {}{region} from {} to {}: {} segment(s) of motion, {:.2} s in all",
|
||||
self.frames_examined,
|
||||
self.displays
|
||||
.iter()
|
||||
.map(ToString::to_string)
|
||||
.collect::<Vec<_>>()
|
||||
.join(", "),
|
||||
time(self.from_video_s),
|
||||
time(self.to_video_s),
|
||||
self.segments.len(),
|
||||
self.active_s(),
|
||||
);
|
||||
for segment in &self.segments {
|
||||
let events = if segment.events.is_empty() {
|
||||
String::new()
|
||||
} else {
|
||||
format!(
|
||||
" events {}",
|
||||
segment
|
||||
.events
|
||||
.iter()
|
||||
.map(|seq| format!("#{seq}"))
|
||||
.collect::<Vec<_>>()
|
||||
.join(" ")
|
||||
)
|
||||
};
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"display {} {} - {} ({:.2} s, {} frames) peak {:.1}% of the pixels, mean delta {:.1} box {:.0}%,{:.0}% {:.0}%x{:.0}%{events}",
|
||||
segment.display,
|
||||
time(segment.start_video_s),
|
||||
time(segment.end_video_s),
|
||||
segment.duration_s,
|
||||
segment.frames,
|
||||
segment.peak_share * 100.0,
|
||||
segment.peak_mean_delta,
|
||||
segment.bbox[0] * 100.0,
|
||||
segment.bbox[1] * 100.0,
|
||||
segment.bbox[2] * 100.0,
|
||||
segment.bbox[3] * 100.0,
|
||||
);
|
||||
}
|
||||
for gap in &self.gaps {
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"display {} has no video from {} to {}",
|
||||
gap.display,
|
||||
time(gap.from_video_s),
|
||||
time(gap.to_video_s)
|
||||
);
|
||||
}
|
||||
out
|
||||
}
|
||||
}
|
||||
|
||||
/// The width to decode a frame at so that a region of fractional width `fraction` is at least
|
||||
/// [`REGION_PIXELS`] across.
|
||||
pub(super) fn scale_for(fraction: f64) -> Scale {
|
||||
let wanted = (REGION_PIXELS / fraction.max(1e-6)).ceil();
|
||||
Scale::Width((wanted as u32).clamp(WHOLE_WIDTH, MAX_WIDTH))
|
||||
}
|
||||
|
||||
impl Analysis {
|
||||
/// The frame of a display in global points, `[x, y, width, height]`, if the session knows it.
|
||||
pub(super) fn display_frame(&self, display: u32) -> Option<[f64; 4]> {
|
||||
self.manifest
|
||||
.displays
|
||||
.iter()
|
||||
.find(|entry| entry.result.display.id == display)
|
||||
.map(|entry| entry.result.display.frame)
|
||||
}
|
||||
|
||||
/// Turns a crop in points into the fractions of `display`.
|
||||
pub(super) fn crop_fractions(&self, display: u32, crop: &Crop) -> eyre::Result<[f64; 4]> {
|
||||
let frame = self.display_frame(display).ok_or_else(|| {
|
||||
envelope::usage(format!(
|
||||
"display {display} has no known frame, so a region in points cannot be placed on it"
|
||||
))
|
||||
})?;
|
||||
crop.fractions(frame)
|
||||
}
|
||||
|
||||
/// Finds the segments of activity on each display of `options`, in the whole display or in
|
||||
/// `options.crop`, with `sensitivity` as for [`Analysis::changes`].
|
||||
///
|
||||
/// Stops early, without an error, once `options.cancel` is set; [`MotionData::frames_examined`]
|
||||
/// says how far it got. A display with no video in the range contributes no segments and is
|
||||
/// reported in [`MotionData::gaps`].
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// Returns an error if:
|
||||
///
|
||||
/// - `sensitivity` is not above zero (the usage exit kind);
|
||||
/// - the crop does not overlap a display, or a display's frame is unknown (the usage kind);
|
||||
/// - a video file cannot be resolved inside the session, opened or decoded, or the host has no
|
||||
/// decoder.
|
||||
///
|
||||
/// # Example
|
||||
///
|
||||
/// This needs recorded video, so it is not run here. `tests/video.rs` runs it through the
|
||||
/// `motion` command on sessions with numbered videos.
|
||||
///
|
||||
/// ```rust,no_run
|
||||
/// use komorebi_recorder::analysis::Analysis;
|
||||
/// use komorebi_recorder::analysis::motion::ScanOptions;
|
||||
/// use komorebi_recorder::analysis::pixels::Crop;
|
||||
/// use komorebi_recorder::analysis::video::SessionVideo;
|
||||
///
|
||||
/// let analysis = Analysis::open(std::path::Path::new("session-dir"))?;
|
||||
/// let video = SessionVideo::new(&analysis.manifest, analysis.guard());
|
||||
/// let data = analysis.motion(
|
||||
/// &video,
|
||||
/// &ScanOptions {
|
||||
/// displays: &video.displays(),
|
||||
/// from_s: 0.0,
|
||||
/// to_s: 60.0,
|
||||
/// crop: Some(Crop::parse("100,100,800,600")?),
|
||||
/// cancel: None,
|
||||
/// },
|
||||
/// 1.0,
|
||||
/// )?;
|
||||
/// print!("{}", data.to_text());
|
||||
/// # Ok::<(), Box<dyn std::error::Error>>(())
|
||||
/// ```
|
||||
pub fn motion(
|
||||
&self,
|
||||
video: &SessionVideo,
|
||||
options: &ScanOptions,
|
||||
sensitivity: f64,
|
||||
) -> eyre::Result<MotionData> {
|
||||
if sensitivity <= 0.0 {
|
||||
return Err(envelope::usage("--sensitivity must be above zero"));
|
||||
}
|
||||
|
||||
let threshold = super::changes::SHARE_THRESHOLD / sensitivity;
|
||||
let mut segments = Vec::new();
|
||||
let mut examined = 0;
|
||||
for display in options.displays {
|
||||
let (crop, scale) = match &options.crop {
|
||||
Some(crop) => {
|
||||
let fractions = self.crop_fractions(*display, crop)?;
|
||||
(Some(fractions), scale_for(fractions[2]))
|
||||
}
|
||||
None => (None, Scale::Width(WHOLE_WIDTH)),
|
||||
};
|
||||
let tracked = track(
|
||||
video,
|
||||
*display,
|
||||
options.from_s,
|
||||
options.to_s,
|
||||
&Tracking {
|
||||
threshold,
|
||||
crop,
|
||||
scale,
|
||||
cancel: options.cancel,
|
||||
},
|
||||
)?;
|
||||
examined += tracked.examined;
|
||||
|
||||
let frame_box = self.display_frame(*display);
|
||||
for span in tracked.spans {
|
||||
let bbox = fractions_of(span.bounds, tracked.dimensions);
|
||||
segments.push(MotionSegment {
|
||||
display: *display,
|
||||
start_video_s: span.start,
|
||||
end_video_s: span.end,
|
||||
duration_s: span.end - span.start,
|
||||
frames: span.frames,
|
||||
peak_share: span.peak,
|
||||
peak_mean_delta: span.peak_delta,
|
||||
bbox,
|
||||
bbox_points: points_of(frame_box, bbox),
|
||||
events: self.events_between(span.start, span.end),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
segments.sort_by(|a, b| {
|
||||
a.start_video_s
|
||||
.total_cmp(&b.start_video_s)
|
||||
.then(a.display.cmp(&b.display))
|
||||
});
|
||||
|
||||
Ok(MotionData {
|
||||
from_video_s: options.from_s,
|
||||
to_video_s: options.to_s,
|
||||
displays: options.displays.to_vec(),
|
||||
region: options.crop,
|
||||
sensitivity,
|
||||
frames_examined: examined,
|
||||
segments,
|
||||
gaps: self.video_gaps(options.displays, options.from_s, options.to_s),
|
||||
})
|
||||
}
|
||||
|
||||
/// The `seq` of each event received within a quarter of a second of `start_s..=end_s`.
|
||||
fn events_between(&self, start_s: f64, end_s: f64) -> Vec<u64> {
|
||||
self.events()
|
||||
.iter()
|
||||
.filter(|event| {
|
||||
let at = self.video_s(event.recv_mach_ns);
|
||||
at >= start_s - 0.25 && at <= end_s + 0.25
|
||||
})
|
||||
.map(|event| event.seq)
|
||||
.collect()
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn a_narrow_region_is_decoded_wider_but_never_past_the_limit() {
|
||||
assert_eq!(scale_for(1.0), Scale::Width(240));
|
||||
assert_eq!(scale_for(0.5), Scale::Width(480));
|
||||
assert_eq!(scale_for(0.0001), Scale::Width(MAX_WIDTH));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn active_time_adds_up_the_segments() {
|
||||
let segment = |start: f64, end: f64| MotionSegment {
|
||||
display: 1,
|
||||
start_video_s: start,
|
||||
end_video_s: end,
|
||||
duration_s: end - start,
|
||||
frames: 1,
|
||||
peak_share: 0.0,
|
||||
peak_mean_delta: 0.0,
|
||||
bbox: [0.0; 4],
|
||||
bbox_points: None,
|
||||
events: vec![],
|
||||
};
|
||||
let data = MotionData {
|
||||
from_video_s: 0.0,
|
||||
to_video_s: 10.0,
|
||||
displays: vec![1],
|
||||
region: None,
|
||||
sensitivity: 1.0,
|
||||
frames_examined: 0,
|
||||
segments: vec![segment(1.0, 1.5), segment(3.0, 4.0)],
|
||||
gaps: vec![],
|
||||
};
|
||||
assert_eq!(data.active_s(), 1.5);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,438 @@
|
||||
//! Pixel arithmetic shared by the commands that look at frames: brightness, cropping to a region
|
||||
//! of a display, and comparing two pictures.
|
||||
//!
|
||||
//! Nothing here decodes video, so it can be tested on synthetic frames. The commands that use it
|
||||
//! are `changes`, `motion`, `framediff` and `blank` ([`super::changes`], [`super::motion`],
|
||||
//! [`super::framediff`], [`super::blank`]).
|
||||
//!
|
||||
//! # Regions
|
||||
//!
|
||||
//! A [`Crop`] is a rectangle in *display points*: the same global coordinates as
|
||||
//! [`ChangeSpan::bbox_points`](super::changes::ChangeSpan::bbox_points) and a window's rect in
|
||||
//! komorebi's state. It is turned into fractions of a display's frame ([`Crop::fractions`]) and
|
||||
//! then into pixels of whatever size a frame was decoded at ([`pixel_rect`]), so the same crop
|
||||
//! works at any decode size. A caller that has a window's rect passes it to [`Crop::from_rect`].
|
||||
//!
|
||||
//! # Example
|
||||
//!
|
||||
//! ```rust
|
||||
//! use komorebi_recorder::analysis::pixels::{Crop, pixel_rect};
|
||||
//!
|
||||
//! // the right half of a display whose frame is 1000 x 500 points at (100, 0)
|
||||
//! let crop = Crop::parse("600,0,500,500")?;
|
||||
//! let fractions = crop.fractions([100.0, 0.0, 1000.0, 500.0])?;
|
||||
//! assert_eq!(fractions, [0.5, 0.0, 0.5, 1.0]);
|
||||
//!
|
||||
//! // in a frame decoded at 240 x 120 pixels
|
||||
//! let rect = pixel_rect(fractions, 240, 120);
|
||||
//! assert_eq!((rect.x, rect.y, rect.width, rect.height), (120, 0, 120, 120));
|
||||
//! # Ok::<(), Box<dyn std::error::Error>>(())
|
||||
//! ```
|
||||
|
||||
use super::envelope;
|
||||
use color_eyre::eyre;
|
||||
use schemars::JsonSchema;
|
||||
use serde::Serialize;
|
||||
|
||||
/// A rectangle in global display points: `x`, `y`, `width`, `height`.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Serialize, JsonSchema)]
|
||||
pub struct Crop {
|
||||
/// Left edge, in points.
|
||||
pub x: f64,
|
||||
/// Top edge, in points.
|
||||
pub y: f64,
|
||||
/// Width, in points; above zero.
|
||||
pub width: f64,
|
||||
/// Height, in points; above zero.
|
||||
pub height: f64,
|
||||
}
|
||||
|
||||
impl Crop {
|
||||
/// Returns the crop for a rect of `[x, y, width, height]`, such as a window's rect.
|
||||
#[must_use]
|
||||
pub fn from_rect(rect: [f64; 4]) -> Self {
|
||||
Self {
|
||||
x: rect[0],
|
||||
y: rect[1],
|
||||
width: rect[2],
|
||||
height: rect[3],
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns the rect as `[x, y, width, height]`.
|
||||
#[must_use]
|
||||
pub fn rect(&self) -> [f64; 4] {
|
||||
[self.x, self.y, self.width, self.height]
|
||||
}
|
||||
|
||||
/// Parses `x,y,w,h` (numbers, in points).
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// Returns a usage error if the text is not four numbers or the size is not above zero.
|
||||
///
|
||||
/// # Example
|
||||
///
|
||||
/// ```rust
|
||||
/// use komorebi_recorder::analysis::pixels::Crop;
|
||||
///
|
||||
/// assert_eq!(Crop::parse("10, 20,300,40")?.rect(), [10.0, 20.0, 300.0, 40.0]);
|
||||
/// assert!(Crop::parse("1,2,3").is_err());
|
||||
/// assert!(Crop::parse("1,2,0,4").is_err());
|
||||
/// # Ok::<(), Box<dyn std::error::Error>>(())
|
||||
/// ```
|
||||
pub fn parse(text: &str) -> eyre::Result<Self> {
|
||||
let numbers: Vec<f64> = text
|
||||
.split(',')
|
||||
.map(|part| part.trim().parse::<f64>())
|
||||
.collect::<Result<_, _>>()
|
||||
.map_err(|_| {
|
||||
envelope::usage(format!("--region wants x,y,w,h in points, not {text:?}"))
|
||||
})?;
|
||||
let [x, y, width, height] = numbers[..] else {
|
||||
return Err(envelope::usage(format!(
|
||||
"--region wants four numbers, x,y,w,h, not {text:?}"
|
||||
)));
|
||||
};
|
||||
if !(width > 0.0 && height > 0.0 && x.is_finite() && y.is_finite()) {
|
||||
return Err(envelope::usage(
|
||||
"--region needs a width and height above zero",
|
||||
));
|
||||
}
|
||||
|
||||
Ok(Self {
|
||||
x,
|
||||
y,
|
||||
width,
|
||||
height,
|
||||
})
|
||||
}
|
||||
|
||||
/// Returns the crop as fractions of a display whose frame is `frame` (`[x, y, width, height]`
|
||||
/// in points): `[x, y, width, height]`, each from 0 to 1. The part of the crop outside the
|
||||
/// display is cut off.
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// Returns a usage error if the frame has no size or the crop does not overlap the display.
|
||||
pub fn fractions(&self, frame: [f64; 4]) -> eyre::Result<[f64; 4]> {
|
||||
if frame[2] <= 0.0 || frame[3] <= 0.0 {
|
||||
return Err(envelope::usage("the display's frame has no size"));
|
||||
}
|
||||
|
||||
let left = ((self.x - frame[0]) / frame[2]).clamp(0.0, 1.0);
|
||||
let top = ((self.y - frame[1]) / frame[3]).clamp(0.0, 1.0);
|
||||
let right = ((self.x + self.width - frame[0]) / frame[2]).clamp(0.0, 1.0);
|
||||
let bottom = ((self.y + self.height - frame[1]) / frame[3]).clamp(0.0, 1.0);
|
||||
if right <= left || bottom <= top {
|
||||
return Err(envelope::usage(format!(
|
||||
"the region {},{},{},{} is outside the display ({},{} {}x{} points)",
|
||||
self.x, self.y, self.width, self.height, frame[0], frame[1], frame[2], frame[3]
|
||||
)));
|
||||
}
|
||||
|
||||
Ok([left, top, right - left, bottom - top])
|
||||
}
|
||||
}
|
||||
|
||||
/// A rectangle of pixels in a decoded frame.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub struct PixelRect {
|
||||
/// Left edge.
|
||||
pub x: u32,
|
||||
/// Top edge.
|
||||
pub y: u32,
|
||||
/// Width, at least 1.
|
||||
pub width: u32,
|
||||
/// Height, at least 1.
|
||||
pub height: u32,
|
||||
}
|
||||
|
||||
/// Converts fractions of a frame to the pixels of a `width` x `height` frame. The rect is at
|
||||
/// least one pixel and never leaves the frame.
|
||||
#[must_use]
|
||||
pub fn pixel_rect(fractions: [f64; 4], width: u32, height: u32) -> PixelRect {
|
||||
let (w, h) = (width.max(1), height.max(1));
|
||||
let x = ((fractions[0] * f64::from(w)).floor() as u32).min(w - 1);
|
||||
let y = ((fractions[1] * f64::from(h)).floor() as u32).min(h - 1);
|
||||
let right = (((fractions[0] + fractions[2]) * f64::from(w)).ceil() as u32).clamp(x + 1, w);
|
||||
let bottom = (((fractions[1] + fractions[3]) * f64::from(h)).ceil() as u32).clamp(y + 1, h);
|
||||
PixelRect {
|
||||
x,
|
||||
y,
|
||||
width: right - x,
|
||||
height: bottom - y,
|
||||
}
|
||||
}
|
||||
|
||||
/// The brightness (Rec. 601 weights, integers) of every pixel of a BGRA picture.
|
||||
///
|
||||
/// # Example
|
||||
///
|
||||
/// ```rust
|
||||
/// use komorebi_recorder::analysis::pixels::luma;
|
||||
///
|
||||
/// // one white pixel, one black, one pure green (BGRA order)
|
||||
/// assert_eq!(luma(&[255, 255, 255, 255, 0, 0, 0, 255, 0, 255, 0, 255]), [255, 0, 149]);
|
||||
/// ```
|
||||
#[must_use]
|
||||
pub fn luma(bgra: &[u8]) -> Vec<u8> {
|
||||
bgra.as_chunks::<4>()
|
||||
.0
|
||||
.iter()
|
||||
.map(|pixel| {
|
||||
((u32::from(pixel[2]) * 299 + u32::from(pixel[1]) * 587 + u32::from(pixel[0]) * 114)
|
||||
/ 1000) as u8
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// The brightness of `rect` of a picture whose rows are `width` pixels, row by row.
|
||||
#[must_use]
|
||||
pub fn crop_luma(luma: &[u8], width: u32, rect: PixelRect) -> Vec<u8> {
|
||||
let mut out = Vec::with_capacity(rect.width as usize * rect.height as usize);
|
||||
for row in rect.y..rect.y + rect.height {
|
||||
let start = row as usize * width as usize + rect.x as usize;
|
||||
if let Some(line) = luma.get(start..start + rect.width as usize) {
|
||||
out.extend_from_slice(line);
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// The BGRA bytes of `rect` of a BGRA picture whose rows are `width` pixels.
|
||||
#[must_use]
|
||||
pub fn crop_bgra(bgra: &[u8], width: u32, rect: PixelRect) -> Vec<u8> {
|
||||
let mut out = Vec::with_capacity(rect.width as usize * rect.height as usize * 4);
|
||||
for row in rect.y..rect.y + rect.height {
|
||||
let start = (row as usize * width as usize + rect.x as usize) * 4;
|
||||
if let Some(line) = bgra.get(start..start + rect.width as usize * 4) {
|
||||
out.extend_from_slice(line);
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// How two same-sized brightness pictures differ.
|
||||
#[derive(Debug, Clone, Copy, PartialEq)]
|
||||
pub struct Delta {
|
||||
/// How many pixels moved by more than the threshold.
|
||||
pub changed: usize,
|
||||
/// How many pixels were compared.
|
||||
pub total: usize,
|
||||
/// The mean of the absolute brightness difference over all pixels, 0 to 255.
|
||||
pub mean: f64,
|
||||
/// The largest absolute brightness difference of any pixel.
|
||||
pub peak: u8,
|
||||
/// The box around the changed pixels, `(min_x, min_y, max_x, max_y)` in pixels of the
|
||||
/// pictures, or `None` if nothing changed.
|
||||
pub bounds: Option<(u32, u32, u32, u32)>,
|
||||
}
|
||||
|
||||
impl Delta {
|
||||
/// The share of pixels that changed, 0 to 1.
|
||||
#[must_use]
|
||||
pub fn share(&self) -> f64 {
|
||||
self.changed as f64 / self.total.max(1) as f64
|
||||
}
|
||||
}
|
||||
|
||||
/// Compares two brightness pictures `width` pixels wide. A pixel has changed if its brightness
|
||||
/// moved by more than `threshold`. Pictures of different lengths are compared as far as the
|
||||
/// shorter goes.
|
||||
///
|
||||
/// # Example
|
||||
///
|
||||
/// ```rust
|
||||
/// use komorebi_recorder::analysis::pixels::delta;
|
||||
///
|
||||
/// // a 2 x 2 picture where the bottom right pixel went from black to white
|
||||
/// let found = delta(&[0, 0, 0, 0], &[0, 0, 0, 255], 2, 24);
|
||||
/// assert_eq!(found.changed, 1);
|
||||
/// assert_eq!(found.share(), 0.25);
|
||||
/// assert_eq!(found.peak, 255);
|
||||
/// assert_eq!(found.bounds, Some((1, 1, 1, 1)));
|
||||
/// ```
|
||||
#[must_use]
|
||||
pub fn delta(before: &[u8], after: &[u8], width: u32, threshold: i32) -> Delta {
|
||||
let (mut changed, mut sum, mut peak) = (0_usize, 0_u64, 0_u8);
|
||||
let (mut min_x, mut min_y, mut max_x, mut max_y) = (u32::MAX, u32::MAX, 0_u32, 0_u32);
|
||||
let width = width.max(1);
|
||||
for (index, (a, b)) in before.iter().zip(after).enumerate() {
|
||||
let difference = (i32::from(*a) - i32::from(*b)).unsigned_abs() as u8;
|
||||
sum += u64::from(difference);
|
||||
peak = peak.max(difference);
|
||||
if i32::from(difference) > threshold {
|
||||
let (x, y) = (index as u32 % width, index as u32 / width);
|
||||
changed += 1;
|
||||
min_x = min_x.min(x);
|
||||
max_x = max_x.max(x);
|
||||
min_y = min_y.min(y);
|
||||
max_y = max_y.max(y);
|
||||
}
|
||||
}
|
||||
|
||||
let total = before.len().min(after.len());
|
||||
Delta {
|
||||
changed,
|
||||
total,
|
||||
mean: sum as f64 / total.max(1) as f64,
|
||||
peak,
|
||||
bounds: (changed > 0).then_some((min_x, min_y, max_x, max_y)),
|
||||
}
|
||||
}
|
||||
|
||||
/// How even a patch of the screen is.
|
||||
#[derive(Debug, Clone, Copy, PartialEq)]
|
||||
pub struct Evenness {
|
||||
/// The variance of brightness over the patch.
|
||||
pub variance: f64,
|
||||
/// The mean colour, `[red, green, blue]`.
|
||||
pub rgb: [u8; 3],
|
||||
}
|
||||
|
||||
/// Measures a BGRA patch: the variance of its brightness and its mean colour.
|
||||
///
|
||||
/// # Example
|
||||
///
|
||||
/// ```rust
|
||||
/// use komorebi_recorder::analysis::pixels::evenness;
|
||||
///
|
||||
/// let white = [255_u8; 16]; // four white pixels
|
||||
/// let found = evenness(&white);
|
||||
/// assert_eq!(found.variance, 0.0);
|
||||
/// assert_eq!(found.rgb, [255, 255, 255]);
|
||||
/// ```
|
||||
#[must_use]
|
||||
pub fn evenness(bgra: &[u8]) -> Evenness {
|
||||
let pixels = bgra.as_chunks::<4>().0;
|
||||
let count = pixels.len().max(1) as f64;
|
||||
let (mut r, mut g, mut b) = (0_u64, 0_u64, 0_u64);
|
||||
let (mut sum, mut square) = (0_f64, 0_f64);
|
||||
for pixel in pixels {
|
||||
r += u64::from(pixel[2]);
|
||||
g += u64::from(pixel[1]);
|
||||
b += u64::from(pixel[0]);
|
||||
let l = f64::from(
|
||||
(u32::from(pixel[2]) * 299 + u32::from(pixel[1]) * 587 + u32::from(pixel[0]) * 114)
|
||||
/ 1000,
|
||||
);
|
||||
sum += l;
|
||||
square += l * l;
|
||||
}
|
||||
|
||||
let mean = sum / count;
|
||||
Evenness {
|
||||
variance: (square / count - mean * mean).max(0.0),
|
||||
rgb: [
|
||||
(r as f64 / count).round() as u8,
|
||||
(g as f64 / count).round() as u8,
|
||||
(b as f64 / count).round() as u8,
|
||||
],
|
||||
}
|
||||
}
|
||||
|
||||
/// Names a colour a person would: `white`, `black` or `gray` when the channels agree, otherwise
|
||||
/// the hex code (`#1a2b3c`).
|
||||
///
|
||||
/// # Example
|
||||
///
|
||||
/// ```rust
|
||||
/// use komorebi_recorder::analysis::pixels::colour_name;
|
||||
///
|
||||
/// assert_eq!(colour_name([250, 252, 255]), "white");
|
||||
/// assert_eq!(colour_name([3, 3, 3]), "black");
|
||||
/// assert_eq!(colour_name([128, 128, 128]), "gray");
|
||||
/// assert_eq!(colour_name([200, 0, 16]), "#c80010");
|
||||
/// ```
|
||||
#[must_use]
|
||||
pub fn colour_name(rgb: [u8; 3]) -> String {
|
||||
let (low, high) = (
|
||||
rgb.iter().copied().min().unwrap_or(0),
|
||||
rgb.iter().copied().max().unwrap_or(0),
|
||||
);
|
||||
if high - low <= 12 {
|
||||
let mean = (u32::from(rgb[0]) + u32::from(rgb[1]) + u32::from(rgb[2])) / 3;
|
||||
return match mean {
|
||||
224.. => "white",
|
||||
..32 => "black",
|
||||
_ => "gray",
|
||||
}
|
||||
.to_string();
|
||||
}
|
||||
|
||||
format!("#{:02x}{:02x}{:02x}", rgb[0], rgb[1], rgb[2])
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn a_crop_is_cut_off_at_the_display_and_refused_outside_it() {
|
||||
let frame = [0.0, 0.0, 100.0, 100.0];
|
||||
let crop = Crop::from_rect([50.0, -20.0, 100.0, 40.0]);
|
||||
assert_eq!(
|
||||
crop.fractions(frame).expect("overlaps"),
|
||||
[0.5, 0.0, 0.5, 0.2]
|
||||
);
|
||||
assert!(
|
||||
Crop::from_rect([200.0, 0.0, 10.0, 10.0])
|
||||
.fractions(frame)
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_pixel_rect_is_at_least_one_pixel_and_inside_the_frame() {
|
||||
let rect = pixel_rect([0.999, 0.999, 0.0005, 0.0005], 10, 10);
|
||||
assert_eq!((rect.x, rect.y, rect.width, rect.height), (9, 9, 1, 1));
|
||||
let whole = pixel_rect([0.0, 0.0, 1.0, 1.0], 7, 5);
|
||||
assert_eq!((whole.width, whole.height), (7, 5));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cropping_takes_the_rows_of_the_rect() {
|
||||
let picture: Vec<u8> = (0..16).collect(); // 4 x 4
|
||||
let rect = PixelRect {
|
||||
x: 1,
|
||||
y: 2,
|
||||
width: 2,
|
||||
height: 2,
|
||||
};
|
||||
assert_eq!(crop_luma(&picture, 4, rect), [9, 10, 13, 14]);
|
||||
let bgra: Vec<u8> = (0..64).collect();
|
||||
assert_eq!(
|
||||
crop_bgra(&bgra, 4, rect)[..8],
|
||||
[36, 37, 38, 39, 40, 41, 42, 43]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_small_change_under_the_threshold_is_not_a_changed_pixel_but_is_in_the_mean() {
|
||||
let found = delta(&[100, 100], &[110, 100], 2, 24);
|
||||
assert_eq!(found.changed, 0);
|
||||
assert_eq!(found.bounds, None);
|
||||
assert_eq!(found.mean, 5.0);
|
||||
assert_eq!(found.peak, 10);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_box_of_change_covers_every_changed_pixel() {
|
||||
let before = vec![0; 16];
|
||||
let mut after = vec![0; 16];
|
||||
after[1] = 255; // (1, 0)
|
||||
after[14] = 255; // (2, 3)
|
||||
let found = delta(&before, &after, 4, 24);
|
||||
assert_eq!(found.bounds, Some((1, 0, 2, 3)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn uniform_patches_have_no_variance_and_noisy_ones_do() {
|
||||
assert_eq!(
|
||||
evenness(&[10, 20, 30, 255, 10, 20, 30, 255]).rgb,
|
||||
[30, 20, 10]
|
||||
);
|
||||
assert!(evenness(&[0, 0, 0, 255, 255, 255, 255, 255]).variance > 10_000.0);
|
||||
}
|
||||
}
|
||||
@@ -15,12 +15,16 @@ use super::answers::DeletedData;
|
||||
use super::answers::DiffData;
|
||||
use super::answers::EventsData;
|
||||
use super::answers::StateData;
|
||||
use super::blank::BlankData;
|
||||
use super::brief::BriefData;
|
||||
use super::changes::ChangesData;
|
||||
use super::code::CodeData;
|
||||
use super::compare::CompareData;
|
||||
use super::envelope::Envelope;
|
||||
use super::framediff::FrameDiffData;
|
||||
use super::history::HistoryData;
|
||||
use super::logq::LogqData;
|
||||
use super::motion::MotionData;
|
||||
use super::overview::OverviewData;
|
||||
use super::search::SearchData;
|
||||
use super::sheet::SheetData;
|
||||
@@ -110,6 +114,26 @@ pub const ALL: &[Entry] = &[
|
||||
about: "`sheet`: a contact sheet image and where each frame is on it",
|
||||
schema: of::<SheetData>,
|
||||
},
|
||||
Entry {
|
||||
name: "motion/1",
|
||||
about: "`motion`: activity segments of a display or a region of it, with peak share and brightness delta",
|
||||
schema: of::<MotionData>,
|
||||
},
|
||||
Entry {
|
||||
name: "framediff/1",
|
||||
about: "`framediff`: how two frames differ: changed share, brightness delta, box of change",
|
||||
schema: of::<FrameDiffData>,
|
||||
},
|
||||
Entry {
|
||||
name: "blank/1",
|
||||
about: "`blank`: stretches where a region was blank (one colour) or frozen",
|
||||
schema: of::<BlankData>,
|
||||
},
|
||||
Entry {
|
||||
name: "compare/1",
|
||||
about: "`compare`: two recordings side by side with the difference of each measure",
|
||||
schema: of::<CompareData>,
|
||||
},
|
||||
Entry {
|
||||
name: "annotation/1",
|
||||
about: "`annotate add|edit`: the annotation as it now reads",
|
||||
|
||||
@@ -181,14 +181,14 @@ pub struct SheetData {
|
||||
pub skipped: Vec<Skipped>,
|
||||
}
|
||||
|
||||
struct Canvas {
|
||||
pub(super) struct Canvas {
|
||||
width: u32,
|
||||
height: u32,
|
||||
rgba: Vec<u8>,
|
||||
}
|
||||
|
||||
impl Canvas {
|
||||
fn new(width: u32, height: u32) -> Self {
|
||||
pub(super) fn new(width: u32, height: u32) -> Self {
|
||||
let mut rgba = Vec::with_capacity(width as usize * height as usize * 4);
|
||||
for _ in 0..width as usize * height as usize {
|
||||
rgba.extend_from_slice(&[BACKGROUND[0], BACKGROUND[1], BACKGROUND[2], 255]);
|
||||
@@ -200,7 +200,7 @@ impl Canvas {
|
||||
}
|
||||
}
|
||||
|
||||
fn put(&mut self, x: i64, y: i64, color: [u8; 3], alpha: f32) {
|
||||
pub(super) fn put(&mut self, x: i64, y: i64, color: [u8; 3], alpha: f32) {
|
||||
if x < 0 || y < 0 || x >= i64::from(self.width) || y >= i64::from(self.height) {
|
||||
return;
|
||||
}
|
||||
@@ -220,7 +220,15 @@ impl Canvas {
|
||||
}
|
||||
}
|
||||
|
||||
fn border(&mut self, x: u32, y: u32, width: u32, height: u32, thickness: u32, color: [u8; 3]) {
|
||||
pub(super) fn border(
|
||||
&mut self,
|
||||
x: u32,
|
||||
y: u32,
|
||||
width: u32,
|
||||
height: u32,
|
||||
thickness: u32,
|
||||
color: [u8; 3],
|
||||
) {
|
||||
self.rectangle(x, y, width, thickness, color);
|
||||
self.rectangle(x, y + height - thickness, width, thickness, color);
|
||||
self.rectangle(x, y, thickness, height, color);
|
||||
@@ -267,7 +275,7 @@ impl Canvas {
|
||||
}
|
||||
}
|
||||
|
||||
fn save(&self, path: &Path) -> eyre::Result<()> {
|
||||
pub(super) fn save(&self, path: &Path) -> eyre::Result<()> {
|
||||
if let Some(parent) = path.parent() {
|
||||
std::fs::create_dir_all(parent)
|
||||
.wrap_err_with(|| format!("could not create {}", parent.display()))?;
|
||||
|
||||
@@ -47,7 +47,11 @@ pub use query::LocateArgs;
|
||||
pub use query::RangeArgs;
|
||||
pub use query::SummaryArgs;
|
||||
pub use record::RecordArgs;
|
||||
pub use video::BlankArgs;
|
||||
pub use video::ChangesArgs;
|
||||
pub use video::CompareArgs;
|
||||
pub use video::FramediffArgs;
|
||||
pub use video::MotionArgs;
|
||||
pub use video::SheetArgs;
|
||||
pub use viewer_command::ViewArgs;
|
||||
|
||||
@@ -116,6 +120,14 @@ pub enum Command {
|
||||
Trace(TraceArgs),
|
||||
/// When and where on each display the picture changed
|
||||
Changes(ChangesArgs),
|
||||
/// When, and how hard, a display or a region of it was moving
|
||||
Motion(MotionArgs),
|
||||
/// How two frames differ: changed share, brightness delta, box of change, optional PNG
|
||||
Framediff(FramediffArgs),
|
||||
/// When a region was blank (one colour, such as white) or frozen
|
||||
Blank(BlankArgs),
|
||||
/// Two recordings side by side: frames, motion, findings, annotations
|
||||
Compare(CompareArgs),
|
||||
/// A contact sheet image of frames, with the times, events and bug region marked
|
||||
Sheet(SheetArgs),
|
||||
/// Open a recording in a window: play and scrub the video with every event on a timeline
|
||||
@@ -189,6 +201,10 @@ impl Command {
|
||||
Self::Window(args) => args.json,
|
||||
Self::Changes(args) => args.json,
|
||||
Self::Sheet(args) => args.json,
|
||||
Self::Motion(args) => args.json,
|
||||
Self::Framediff(args) => args.json,
|
||||
Self::Blank(args) => args.json,
|
||||
Self::Compare(args) => args.json,
|
||||
Self::Annotate(args) => args.json(),
|
||||
Self::Annotations(args) => args.json,
|
||||
Self::Doctor(args) => args.json,
|
||||
@@ -251,6 +267,10 @@ pub fn run(command: Command) -> eyre::Result<()> {
|
||||
Command::Changes(args) => video::changes(&args),
|
||||
Command::View(args) => crate::viewer::run(args.session),
|
||||
Command::Sheet(args) => video::sheet(&args),
|
||||
Command::Motion(args) => video::motion(&args),
|
||||
Command::Framediff(args) => video::framediff(&args),
|
||||
Command::Blank(args) => video::blank(&args),
|
||||
Command::Compare(args) => video::compare(&args),
|
||||
Command::Annotate(args) => annotate::annotate(&args),
|
||||
Command::Annotations(args) => annotate::list(&args),
|
||||
Command::Reindex(args) => reindex(&args),
|
||||
|
||||
@@ -4,6 +4,10 @@
|
||||
//! |---|---|---|
|
||||
//! | `changes` | [`Analysis::changes`](crate::analysis::Analysis::changes) | `changes/1` |
|
||||
//! | `sheet` | [`Analysis::sheet`] | `sheet/1` |
|
||||
//! | `motion` | [`Analysis::motion`] | `motion/1` |
|
||||
//! | `framediff` | [`framediff`](crate::analysis::framediff::framediff) | `framediff/1` |
|
||||
//! | `blank` | [`Analysis::blank`] | `blank/1` |
|
||||
//! | `compare` | [`Analysis::compare`] | `compare/1` |
|
||||
//!
|
||||
//! Both work on a range of video time, given as `--from` and `--to` (any time `state --at` takes;
|
||||
//! the default is the whole video) or, instead of those, as `--bug <id>`: a bug region from three
|
||||
@@ -18,9 +22,20 @@
|
||||
|
||||
use super::print_json;
|
||||
use crate::analysis::Analysis;
|
||||
use crate::analysis::blank::DEFAULT_MIN_DURATION_S;
|
||||
use crate::analysis::blank::DEFAULT_VARIANCE;
|
||||
use crate::analysis::blank::SCHEMA_BLANK;
|
||||
use crate::analysis::changes::SCHEMA_CHANGES;
|
||||
use crate::analysis::compare::SCHEMA_COMPARE;
|
||||
use crate::analysis::envelope;
|
||||
use crate::analysis::envelope::Envelope;
|
||||
use crate::analysis::framediff::DEFAULT_WIDTH;
|
||||
use crate::analysis::framediff::FrameAt;
|
||||
use crate::analysis::framediff::FrameDiffOptions;
|
||||
use crate::analysis::framediff::SCHEMA_FRAMEDIFF;
|
||||
use crate::analysis::motion::SCHEMA_MOTION;
|
||||
use crate::analysis::motion::ScanOptions;
|
||||
use crate::analysis::pixels::Crop;
|
||||
use crate::analysis::sheet::Region;
|
||||
use crate::analysis::sheet::SCHEMA_SHEET;
|
||||
use crate::analysis::sheet::SheetOptions;
|
||||
@@ -96,6 +111,102 @@ pub struct SheetArgs {
|
||||
pub json: bool,
|
||||
}
|
||||
|
||||
/// The arguments of `motion`.
|
||||
#[derive(Args, Debug)]
|
||||
pub struct MotionArgs {
|
||||
/// Session directory
|
||||
pub session: PathBuf,
|
||||
/// Only this display (repeatable) [default: every display with video]
|
||||
#[arg(long = "display", value_name = "ID")]
|
||||
pub displays: Vec<u32>,
|
||||
/// Only this part of the display, in display points: x,y,w,h
|
||||
#[arg(long, value_name = "X,Y,W,H")]
|
||||
pub region: Option<String>,
|
||||
/// From this moment [default: the start of the video]
|
||||
#[arg(long, value_name = "TIME")]
|
||||
pub from: Option<String>,
|
||||
/// To this moment [default: the end of the video]
|
||||
#[arg(long, value_name = "TIME")]
|
||||
pub to: Option<String>,
|
||||
/// Above 1 notices smaller changes, below 1 only larger ones
|
||||
#[arg(long, default_value_t = 1.0)]
|
||||
pub sensitivity: f64,
|
||||
/// Machine-readable JSON output
|
||||
#[arg(long)]
|
||||
pub json: bool,
|
||||
}
|
||||
|
||||
/// The arguments of `framediff`.
|
||||
#[derive(Args, Debug)]
|
||||
pub struct FramediffArgs {
|
||||
/// Session directory
|
||||
pub session: PathBuf,
|
||||
/// The first moment
|
||||
#[arg(value_name = "TIME_A")]
|
||||
pub time_a: String,
|
||||
/// The second moment [default with --against: the same as TIME_A]
|
||||
#[arg(value_name = "TIME_B")]
|
||||
pub time_b: Option<String>,
|
||||
/// Compare with the frame of this other recording at TIME_B (resolved in that recording)
|
||||
#[arg(long, value_name = "SESSION")]
|
||||
pub against: Option<PathBuf>,
|
||||
/// The display [default: the first display with video]
|
||||
#[arg(long = "display", value_name = "ID")]
|
||||
pub display: Option<u32>,
|
||||
/// Only this part of the display, in display points: x,y,w,h
|
||||
#[arg(long, value_name = "X,Y,W,H")]
|
||||
pub region: Option<String>,
|
||||
/// Write a PNG of the compared area with the changed pixels in red
|
||||
#[arg(long, value_name = "FILE")]
|
||||
pub png: Option<PathBuf>,
|
||||
/// The width frames are decoded at, in pixels
|
||||
#[arg(long, default_value_t = DEFAULT_WIDTH)]
|
||||
pub width: u32,
|
||||
/// Machine-readable JSON output
|
||||
#[arg(long)]
|
||||
pub json: bool,
|
||||
}
|
||||
|
||||
/// The arguments of `blank`.
|
||||
#[derive(Args, Debug)]
|
||||
pub struct BlankArgs {
|
||||
/// Session directory
|
||||
pub session: PathBuf,
|
||||
/// The region to watch, in display points: x,y,w,h
|
||||
#[arg(long, value_name = "X,Y,W,H")]
|
||||
pub region: String,
|
||||
/// Only this display (repeatable) [default: every display with video]
|
||||
#[arg(long = "display", value_name = "ID")]
|
||||
pub displays: Vec<u32>,
|
||||
/// From this moment [default: the start of the video]
|
||||
#[arg(long, value_name = "TIME")]
|
||||
pub from: Option<String>,
|
||||
/// To this moment [default: the end of the video]
|
||||
#[arg(long, value_name = "TIME")]
|
||||
pub to: Option<String>,
|
||||
/// A region is blank when the variance of its brightness is below this
|
||||
#[arg(long, default_value_t = DEFAULT_VARIANCE)]
|
||||
pub variance: f64,
|
||||
/// Drop stretches shorter than this many milliseconds
|
||||
#[arg(long, value_name = "MS", default_value_t = DEFAULT_MIN_DURATION_S * 1000.0)]
|
||||
pub min_ms: f64,
|
||||
/// Machine-readable JSON output
|
||||
#[arg(long)]
|
||||
pub json: bool,
|
||||
}
|
||||
|
||||
/// The arguments of `compare`.
|
||||
#[derive(Args, Debug)]
|
||||
pub struct CompareArgs {
|
||||
/// The first session directory
|
||||
pub session_a: PathBuf,
|
||||
/// The second session directory
|
||||
pub session_b: PathBuf,
|
||||
/// Machine-readable JSON output
|
||||
#[arg(long)]
|
||||
pub json: bool,
|
||||
}
|
||||
|
||||
/// The video-time range to work on: a bug region, or `--from`/`--to`, or everything.
|
||||
fn range(
|
||||
analysis: &Analysis,
|
||||
@@ -233,3 +344,139 @@ pub(super) fn sheet(args: &SheetArgs) -> eyre::Result<()> {
|
||||
print!("{}", report::sheet_text(&data));
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn region(text: Option<&str>) -> eyre::Result<Option<Crop>> {
|
||||
text.map(Crop::parse).transpose()
|
||||
}
|
||||
|
||||
pub(super) fn motion(args: &MotionArgs) -> eyre::Result<()> {
|
||||
let analysis = Analysis::open(&args.session)?;
|
||||
let video = SessionVideo::new(&analysis.manifest, analysis.guard());
|
||||
let (from, to) = range(&analysis, args.from.as_deref(), args.to.as_deref(), None)?;
|
||||
let shown = displays(&video, &args.displays)?;
|
||||
|
||||
let data = analysis.motion(
|
||||
&video,
|
||||
&ScanOptions {
|
||||
displays: &shown,
|
||||
from_s: from,
|
||||
to_s: to,
|
||||
crop: region(args.region.as_deref())?,
|
||||
cancel: None,
|
||||
},
|
||||
args.sensitivity,
|
||||
)?;
|
||||
if args.json {
|
||||
return print_json(&Envelope::ok(SCHEMA_MOTION, data).warnings(analysis.warnings()));
|
||||
}
|
||||
|
||||
print!("{}", data.to_text());
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(super) fn framediff(args: &FramediffArgs) -> eyre::Result<()> {
|
||||
let analysis = Analysis::open(&args.session)?;
|
||||
let video = SessionVideo::new(&analysis.manifest, analysis.guard());
|
||||
let wanted: Vec<u32> = args.display.into_iter().collect();
|
||||
let display = displays(&video, &wanted)?[0];
|
||||
let options = FrameDiffOptions {
|
||||
crop: region(args.region.as_deref())?,
|
||||
width: args.width,
|
||||
png: args.png.as_deref(),
|
||||
};
|
||||
let first = FrameAt {
|
||||
analysis: &analysis,
|
||||
video: &video,
|
||||
display,
|
||||
time_s: analysis.resolve(&args.time_a)?.video_s,
|
||||
};
|
||||
|
||||
let data = if let Some(path) = &args.against {
|
||||
let other = Analysis::open(path)?;
|
||||
let other_video = SessionVideo::new(&other.manifest, other.guard());
|
||||
let available = other_video.displays();
|
||||
let other_display = if available.contains(&display) {
|
||||
display
|
||||
} else if let [only] = available[..] {
|
||||
only
|
||||
} else {
|
||||
return Err(envelope::not_found(format!(
|
||||
"{} has no display {display}; displays with video: {available:?}",
|
||||
path.display()
|
||||
)));
|
||||
};
|
||||
let time_b = other
|
||||
.resolve(args.time_b.as_deref().unwrap_or(&args.time_a))?
|
||||
.video_s;
|
||||
crate::analysis::framediff::framediff(
|
||||
&first,
|
||||
&FrameAt {
|
||||
analysis: &other,
|
||||
video: &other_video,
|
||||
display: other_display,
|
||||
time_s: time_b,
|
||||
},
|
||||
&options,
|
||||
)?
|
||||
} else {
|
||||
let text = args.time_b.as_deref().ok_or_else(|| {
|
||||
envelope::usage("framediff needs a second time, or --against another session")
|
||||
})?;
|
||||
let time_b = analysis.resolve(text)?.video_s;
|
||||
crate::analysis::framediff::framediff(
|
||||
&first,
|
||||
&FrameAt {
|
||||
time_s: time_b,
|
||||
..first
|
||||
},
|
||||
&options,
|
||||
)?
|
||||
};
|
||||
if args.json {
|
||||
return print_json(&Envelope::ok(SCHEMA_FRAMEDIFF, data).warnings(analysis.warnings()));
|
||||
}
|
||||
|
||||
print!("{}", data.to_text());
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(super) fn blank(args: &BlankArgs) -> eyre::Result<()> {
|
||||
let analysis = Analysis::open(&args.session)?;
|
||||
let video = SessionVideo::new(&analysis.manifest, analysis.guard());
|
||||
let (from, to) = range(&analysis, args.from.as_deref(), args.to.as_deref(), None)?;
|
||||
let shown = displays(&video, &args.displays)?;
|
||||
|
||||
let data = analysis.blank(
|
||||
&video,
|
||||
&ScanOptions {
|
||||
displays: &shown,
|
||||
from_s: from,
|
||||
to_s: to,
|
||||
crop: region(Some(&args.region))?,
|
||||
cancel: None,
|
||||
},
|
||||
args.variance,
|
||||
args.min_ms / 1000.0,
|
||||
)?;
|
||||
if args.json {
|
||||
return print_json(&Envelope::ok(SCHEMA_BLANK, data).warnings(analysis.warnings()));
|
||||
}
|
||||
|
||||
print!("{}", data.to_text());
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(super) fn compare(args: &CompareArgs) -> eyre::Result<()> {
|
||||
let a = Analysis::open(&args.session_a)?;
|
||||
let b = Analysis::open(&args.session_b)?;
|
||||
|
||||
let data = a.compare(&b, None)?;
|
||||
if args.json {
|
||||
let mut warnings = a.warnings();
|
||||
warnings.extend(b.warnings());
|
||||
return print_json(&Envelope::ok(SCHEMA_COMPARE, data).warnings(warnings));
|
||||
}
|
||||
|
||||
print!("{}", data.to_text());
|
||||
Ok(())
|
||||
}
|
||||
@@ -96,6 +96,11 @@ locations as hints.
|
||||
`unmanaged-window` (on screen and not managed: usually right, and komorebi's explanation is
|
||||
quoted), and `trace event:N` shows the facts. Without the probe none of that is knowable, and
|
||||
`overview` says nothing about it.
|
||||
- To measure the picture instead of scripting `ffmpeg`: `motion` (when a display or `--region x,y,w,h` moved,
|
||||
and how hard), `framediff <a> <b>` (what changed between two moments; `--png` shows where),
|
||||
`blank --region x,y,w,h` (when a region went one colour, such as white, or froze) and
|
||||
`compare <a> <b>` (two recordings' frames, motion and findings side by side). Regions are in display
|
||||
points, the same as the `bbox_points` of `changes`. They need a host that can decode the video.
|
||||
- Video time `00:00.000` is the first frame of any display. A display that was unplugged, asleep
|
||||
or whose capture failed has a gap, and `incident` records say why.
|
||||
|
||||
|
||||
@@ -284,3 +284,183 @@ fn a_moment_a_display_had_no_video_is_skipped_not_invented() {
|
||||
]);
|
||||
assert_eq!(huge.status.code(), Some(2));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn motion_finds_the_segments_and_a_region_limits_where_it_looks() {
|
||||
let Some(dir) = session("video-motion") else {
|
||||
return;
|
||||
};
|
||||
|
||||
let answer = json(&["motion", &dir, "--display", "1", "--json"]);
|
||||
assert_eq!(answer["schema"], "motion/1");
|
||||
let segments = answer["data"]["segments"].as_array().expect("segments");
|
||||
assert_eq!(segments.len(), 2, "{segments:#?}");
|
||||
assert!(segments[0]["peak_share"].as_f64().expect("share") > 0.0);
|
||||
assert!(segments[0]["peak_mean_delta"].as_f64().expect("delta") > 0.0);
|
||||
let duration = segments[1]["duration_s"].as_f64().expect("duration");
|
||||
assert!((duration - 0.97).abs() < 0.1, "{duration}");
|
||||
assert_eq!(segments[1]["events"], json!([1]));
|
||||
|
||||
// counting changes the blocks on the right of the picture: the left half of the display (the
|
||||
// test display is 1000 x 800 points) holds still, the right half moves in both segments
|
||||
let left = json(&[
|
||||
"motion",
|
||||
&dir,
|
||||
"--display",
|
||||
"1",
|
||||
"--region",
|
||||
"0,0,500,800",
|
||||
"--json",
|
||||
]);
|
||||
assert_eq!(left["data"]["segments"], json!([]));
|
||||
let right = json(&[
|
||||
"motion",
|
||||
&dir,
|
||||
"--display",
|
||||
"1",
|
||||
"--region",
|
||||
"500,0,500,800",
|
||||
"--json",
|
||||
]);
|
||||
let moved = right["data"]["segments"].as_array().expect("segments");
|
||||
assert_eq!(moved.len(), 2, "{moved:#?}");
|
||||
assert!(moved[0]["bbox_points"][0].as_f64().expect("x") >= 500.0);
|
||||
|
||||
// a region off the display is a usage error, so is a malformed one
|
||||
for bad in ["5000,0,10,10", "1,2,3"] {
|
||||
let output = recorder(&["motion", &dir, "--region", bad]);
|
||||
assert_eq!(output.status.code(), Some(2), "{bad}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn framediff_says_how_much_changed_and_where_and_writes_the_picture() {
|
||||
let Some(dir) = session("video-framediff") else {
|
||||
return;
|
||||
};
|
||||
let png = std::path::Path::new(&dir).join("diff.png");
|
||||
let png_arg = png.display().to_string();
|
||||
|
||||
// the picture is counting at 0.2 s and 0.5 s
|
||||
let moved = json(&[
|
||||
"framediff",
|
||||
&dir,
|
||||
"0.2",
|
||||
"0.5",
|
||||
"--display",
|
||||
"1",
|
||||
"--png",
|
||||
&png_arg,
|
||||
"--json",
|
||||
]);
|
||||
assert_eq!(moved["schema"], "framediff/1");
|
||||
let data = &moved["data"];
|
||||
assert!(data["changed_share"].as_f64().expect("share") > 0.0);
|
||||
assert!(data["peak_delta"].as_u64().expect("peak") > 100);
|
||||
assert!(data["bbox"][0].as_f64().expect("x") >= 0.5);
|
||||
assert!(png.is_file());
|
||||
assert_eq!(&std::fs::read(&png).expect("png")[1..4], b"PNG");
|
||||
|
||||
// it holds still from 1.1 s to 1.8 s, and a recording is the same as itself
|
||||
let still = json(&["framediff", &dir, "1.1", "1.8", "--display", "1", "--json"]);
|
||||
assert_eq!(still["data"]["changed_pixels"], 0);
|
||||
assert!(still["data"]["bbox"].is_null());
|
||||
let same = json(&[
|
||||
"framediff",
|
||||
&dir,
|
||||
"0.5",
|
||||
"--display",
|
||||
"1",
|
||||
"--against",
|
||||
&dir,
|
||||
"--json",
|
||||
]);
|
||||
assert_eq!(same["data"]["changed_pixels"], 0);
|
||||
|
||||
// without a second time or another session there is nothing to compare with
|
||||
assert_eq!(
|
||||
recorder(&["framediff", &dir, "0.5", "--display", "1"])
|
||||
.status
|
||||
.code(),
|
||||
Some(2)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn blank_finds_the_region_going_white_and_the_picture_freezing() {
|
||||
if !testkit::ffmpeg_available() {
|
||||
eprintln!("skipped: ffmpeg is not available to make a test video");
|
||||
return;
|
||||
}
|
||||
|
||||
// a still picture for a second, all white for half a second, the picture again
|
||||
let (session, _) = SessionBuilder::new("video-blank")
|
||||
.video_showing(1, 0.0, 90, 30, |frame| match frame {
|
||||
30..=44 => 0xFFFF,
|
||||
_ => 0x5A5A,
|
||||
})
|
||||
.build();
|
||||
let dir = session.dir.display().to_string();
|
||||
|
||||
let answer = json(&["blank", &dir, "--region", "0,0,1000,800", "--json"]);
|
||||
assert_eq!(answer["schema"], "blank/1");
|
||||
let stretches = answer["data"]["stretches"].as_array().expect("stretches");
|
||||
let blank: Vec<&Value> = stretches
|
||||
.iter()
|
||||
.filter(|stretch| stretch["kind"] == "blank")
|
||||
.collect();
|
||||
assert_eq!(blank.len(), 1, "{stretches:#?}");
|
||||
assert_eq!(blank[0]["colour"], "white");
|
||||
assert!((blank[0]["start_video_s"].as_f64().expect("start") - 1.0).abs() < 0.06);
|
||||
assert!((blank[0]["duration_s"].as_f64().expect("duration") - 0.5).abs() < 0.1);
|
||||
|
||||
// the picture before and after repeats, frame after frame
|
||||
let frozen = stretches
|
||||
.iter()
|
||||
.filter(|stretch| stretch["kind"] == "frozen")
|
||||
.count();
|
||||
assert!(frozen >= 2, "{stretches:#?}");
|
||||
|
||||
// a minimum length drops what is shorter, and a region is required
|
||||
let long = json(&[
|
||||
"blank",
|
||||
&dir,
|
||||
"--region",
|
||||
"0,0,1000,800",
|
||||
"--min-ms",
|
||||
"5000",
|
||||
"--json",
|
||||
]);
|
||||
assert_eq!(long["data"]["stretches"], json!([]));
|
||||
assert_eq!(recorder(&["blank", &dir]).status.code(), Some(2));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn compare_sets_two_recordings_side_by_side() {
|
||||
let Some(dir) = session("video-compare") else {
|
||||
return;
|
||||
};
|
||||
// frames come every 33 ms, except for a hole of 300 ms
|
||||
let mut csv = String::from("index,pts_mach_ns,received_mach_ns\n");
|
||||
let mut at = 1_000_000_000_u64;
|
||||
for index in 0..60 {
|
||||
csv.push_str(&format!("{index},{at},{at}\n"));
|
||||
at += if index == 29 { 300_000_000 } else { 33_000_000 };
|
||||
}
|
||||
std::fs::write(std::path::Path::new(&dir).join("frames-1.csv"), csv).expect("frame log");
|
||||
|
||||
let answer = json(&["compare", &dir, &dir, "--json"]);
|
||||
assert_eq!(answer["schema"], "compare/1");
|
||||
let data = &answer["data"];
|
||||
assert_eq!(data["a"]["frames"], 60);
|
||||
assert!((data["a"]["worst_gap_ms"].as_f64().expect("gap") - 300.0).abs() < 1.0);
|
||||
assert_eq!(data["a"]["late_gaps"], 1);
|
||||
// two on display 1 and one on display 2
|
||||
assert_eq!(data["a"]["motion"]["segments"], 3);
|
||||
assert_eq!(data["deltas"]["frames"], 0);
|
||||
assert_eq!(data["deltas"]["worst_gap_ms"], 0.0);
|
||||
|
||||
let text = recorder(&["compare", &dir, &dir]);
|
||||
assert!(text.status.success());
|
||||
assert!(String::from_utf8_lossy(&text.stdout).contains("worst frame gap"));
|
||||
}
|
||||
Reference in new issue
Block a user