feat(analysis): see what the screen did

When and where on each display the picture changed, found by decoding the video,
and a contact sheet of frames with the times, events and bug region marked. A
display with no video for part of a range has a gap there instead of an invented
frame.
This commit is contained in:
LGUG2Z committed 2026-10-27 22:22:56 -07:00
1 parent 26963391a3
commit 069347060d
12 files changed
+2043

No files matched your search

Generated
+2
View File
@@ -1907,6 +1907,7 @@ checksum = "e2db585e1d738fc771bf08a151420d3ed193d9d895a36df7f6f8a9456b911ddc"
name = "komorebi-recorder"
version = "0.1.0"
dependencies = [
"ab_glyph",
"block2 0.6.2",
"chrono",
"clap",
@@ -1917,6 +1918,7 @@ dependencies = [
"dispatch2",
"dunce",
"eframe",
"epaint_default_fonts",
"libc",
"objc2 0.6.4",
"objc2-av-foundation",
+2
View File
@@ -45,6 +45,8 @@ and `event:12.end` when it finished.
| `window ID\|TEXT` | One window's life: when it appeared, moved, was parked off screen, was really somewhere else, and which event did each |
| `code event:N`, `code log:N` | The line of komorebi's source that wrote a log line, with the code around it |
| `search TEXT` | Where a word appears: komorebi's log, events, incidents, annotations (and `--oslog` for the macOS trace) |
| `changes` | When and where on each display the picture changed, and which events fall inside |
| `sheet` | A contact sheet image of frames with times, events and the bug region marked |
| `annotate`, `annotations` | Add, change and list bug regions, notes and findings |
### Who changed it
+2
View File
@@ -6,6 +6,7 @@ edition.workspace = true
repository.workspace = true
[dependencies]
ab_glyph = "0.2"
chrono = { workspace = true }
clap = { workspace = true }
color-eyre = { workspace = true }
@@ -14,6 +15,7 @@ ctrlc = { version = "3", features = ["termination"] }
dirs = { workspace = true }
dunce = { workspace = true }
eframe = { workspace = true }
epaint_default_fonts = "0.33"
parking_lot = { workspace = true }
png = "0.18"
regex = "1"
+444
View File
@@ -0,0 +1,444 @@
//! When, and where on the screen, the picture changed.
//!
//! komorebi's events say when it thought something happened; the video says when the screen
//! actually did. Comparing consecutive frames finds the stretches where pixels changed, how much
//! of the screen they covered and the box around them, so an investigator can see whether a layout
//! change was instant or took half a second, whether a window jumped, and what else on screen was
//! going on. Where an event falls inside a stretch of change it is named.
//!
//! [`Analysis::changes`] is what the `changes` command prints and what the viewer scans in the
//! background ([`Analysis::changes_until`]) to draw the screen lane of its timeline.
//! [`Analysis::video_gaps`] reports where a display had no video to compare at all.
//!
//! This is about pixels, not about komorebi's state: a [`ChangeSpan`] reports that the picture
//! moved, not what moved. The differences between two komorebi states are in [`super::diff`].
//!
//! # How a change is found
//!
//! Each display is decoded separately and every frame is compared with the one before it:
//!
//! - Frames are scaled to 240 pixels wide and reduced to brightness (Rec. 601 weights).
//! - A pixel has changed if its brightness moved by more than 24 (of 255).
//! - A frame is a change if the share of its pixels that changed is at least 0.05% divided by the
//! `sensitivity` asked for: with a sensitivity of 2, changes half as large are noticed, and with
//! 0.5 only changes twice as large.
//! - Changed frames less than 0.15 s apart belong to one [`ChangeSpan`].
//! - A frame is compared with the one before it on the same display, across the start of a new
//! segment too (a display that came back after being unplugged or asleep), so a screen that
//! looks different after the break is a change at the boundary. A frame of another size than
//! the one before it (a display that changed resolution) is not compared with it.
//!
//! Frames come when the screen changes, not on a schedule, so a display whose video starts less
//! than 0.5 s after the range starts, or stops less than 0.5 s before it ends, is not reported as
//! having a [`Gap`].
//!
//! Decoding needs a video decoder on the host: where there is none, the functions here that read
//! frames return an error saying so, and everything else in a session can still be read.
use super::Analysis;
use super::envelope;
use super::video::DecodedFrame;
use super::video::SessionVideo;
use crate::av::Scale;
use color_eyre::eyre;
use schemars::JsonSchema;
use serde::Serialize;
/// The schema name that identifies a [`ChangesData`] in an envelope (`changes/1`); listed by the
/// `schema` command.
pub const SCHEMA_CHANGES: &str = "changes/1";
/// Frames are compared at this width: enough to see a window move, cheap enough to scan minutes.
const COMPARE_WIDTH: u32 = 240;
/// A pixel counts as changed if its brightness moved by more than this (out of 255).
const PIXEL_THRESHOLD: i32 = 24;
/// A frame is a change if at least this share of its pixels changed, at sensitivity 1.
const SHARE_THRESHOLD: f64 = 0.0005;
/// Frames of one change closer together than this are one change.
const MERGE_GAP_S: f64 = 0.15;
/// A display whose first frame arrives this late, or whose video stops this early, is not
/// reported as having a gap: frames come when the screen changes, not on a schedule.
const MIN_GAP_S: f64 = 0.5;
/// An event this close to a change is said to coincide with it.
const COINCIDE_S: f64 = 0.25;
/// A stretch of video in which the picture of one display changed.
///
/// A span of a single changed frame has `start_video_s == end_video_s`.
#[derive(Debug, Clone, PartialEq, Serialize, JsonSchema)]
pub struct ChangeSpan {
/// The id of the display, the same number as [`Monitor::id`](super::state::Monitor::id) and
/// the `--display` of the recording.
pub display: u32,
/// When the first changed frame was on screen, in seconds of video time.
pub start_video_s: f64,
/// When the last changed frame was on screen, in seconds of video time.
pub end_video_s: f64,
/// Changed frames in the span: not the frames that fall between its start and end.
pub frames: usize,
/// The largest share of the screen that changed between two consecutive frames.
pub peak_share: f64,
/// The box around everything that changed, as fractions of the screen: `[x, y, width, height]`.
/// The fractions are of the size of the last frame decoded for the display.
pub bbox: [f64; 4],
/// The same box in global display points, if the display's frame is known:
/// `[x, y, width, height]`.
pub bbox_points: Option<[f64; 4]>,
/// The `seq` of each komorebi event received from 0.25 s before the span starts to 0.25 s
/// after it ends. An event is received when its handler has finished.
pub events: Vec<u64>,
}
/// A stretch of a range in which a display had no video at all.
///
/// A display has none while it is unplugged or asleep, or after its capture failed.
#[derive(Debug, Clone, PartialEq, Serialize, JsonSchema)]
pub struct Gap {
/// The id of the display.
pub display: u32,
/// Where the gap starts, in seconds of video time.
pub from_video_s: f64,
/// Where the gap ends, in seconds of video time.
pub to_video_s: f64,
}
/// Where the picture changed on each display over a range of the recording: the `data` of the
/// `changes` command.
#[derive(Debug, Clone, PartialEq, Serialize, JsonSchema)]
pub struct ChangesData {
/// The start of the range that was scanned, in seconds of video time.
pub from_video_s: f64,
/// The end of the range that was scanned, in seconds of video time.
pub to_video_s: f64,
/// The displays that were scanned, in the order asked for.
pub displays: Vec<u32>,
/// How many frames were decoded and looked at, over all displays. A scan that was cancelled
/// counts only what it got through.
pub frames_examined: usize,
/// The changes, ordered by start time and then by display.
pub spans: Vec<ChangeSpan>,
/// Stretches of the range where a display had no video at all (unplugged, asleep). See
/// [`Analysis::video_gaps`].
pub gaps: Vec<Gap>,
}
/// The brightness of every pixel of a BGRA frame.
fn luma(frame: &DecodedFrame) -> Vec<u8> {
frame
.frame
.bgra
.as_chunks::<4>()
.0
.iter()
.map(|pixel| {
// Rec. 601 weights in integers
((u32::from(pixel[2]) * 299 + u32::from(pixel[1]) * 587 + u32::from(pixel[0]) * 114)
/ 1000) as u8
})
.collect()
}
struct Open {
start: f64,
end: f64,
frames: usize,
peak: f64,
bounds: (u32, u32, u32, u32),
}
impl Analysis {
/// Finds where the picture changed on each of `displays` in `from_s..=to_s` (seconds of video
/// time).
///
/// `sensitivity` scales how little has to change for a frame to count: the default of the
/// `changes` command is `1.0`, a larger number notices smaller changes and a smaller one only
/// larger changes. How a change is found is described in the [module](self) documentation.
/// A display with no video in the range contributes no spans and is reported in
/// [`ChangesData::gaps`]; so is one that is not in the session at all.
///
/// This decodes every frame of the range, so its cost is that of decoding the video, which was
/// measured at 250 to 970 frames per second on macOS (HEVC, depending on frame size). `video`
/// belongs to the thread that made it (a `SessionVideo` is not `Send`), so a scan on another
/// thread builds a new `SessionVideo`.
///
/// # Errors
///
/// Returns an error if:
///
/// - `sensitivity` is not above zero (the usage exit kind);
/// - a display's video file cannot be resolved inside the session, opened or decoded (missing,
/// truncated or unreadable), or the host has no video decoder.
///
/// # Example
///
/// This needs recorded video, so it is not run here. It is covered by
/// `tests/video.rs::the_picture_changing_is_found_where_it_changed_and_not_where_it_held_still`
/// and `tests/video.rs::a_display_with_no_video_for_part_of_the_range_has_a_gap`, which build
/// sessions with numbered videos and run the `changes` command over them.
///
/// ```rust,no_run
/// use komorebi_recorder::analysis::Analysis;
/// use komorebi_recorder::analysis::guard::SessionPath;
/// use komorebi_recorder::analysis::video::SessionVideo;
///
/// let analysis = Analysis::open(std::path::Path::new("session-dir"))?;
/// let guard = SessionPath::new(&analysis.session.dir)?;
/// let video = SessionVideo::new(&analysis.manifest, &guard);
///
/// let data = analysis.changes(&video, &video.displays(), 0.0, 60.0, 1.0)?;
/// for span in &data.spans {
/// println!(
/// "display {}: {:.2} s to {:.2} s, events {:?}",
/// span.display, span.start_video_s, span.end_video_s, span.events
/// );
/// }
/// # Ok::<(), Box<dyn std::error::Error>>(())
/// ```
pub fn changes(
&self,
video: &SessionVideo,
displays: &[u32],
from_s: f64,
to_s: f64,
sensitivity: f64,
) -> eyre::Result<ChangesData> {
self.changes_until(video, displays, from_s, to_s, sensitivity, None)
}
/// Finds where the picture changed, like [`Analysis::changes`], and stops early with what it
/// has found so far once `cancel` is set: for a scan that runs in the background and whose
/// result is no longer wanted.
///
/// `cancel` is checked once per decoded frame, so a scan stops within a frame of it being set.
/// A cancelled scan is not an error: it returns the spans of what it got through, and
/// [`ChangesData::frames_examined`] says how far that was. With `None` it runs to the end,
/// which is what [`Analysis::changes`] does.
///
/// # Errors
///
/// The same as [`Analysis::changes`].
pub fn changes_until(
&self,
video: &SessionVideo,
displays: &[u32],
from_s: f64,
to_s: f64,
sensitivity: f64,
cancel: Option<&std::sync::atomic::AtomicBool>,
) -> eyre::Result<ChangesData> {
if sensitivity <= 0.0 {
return Err(envelope::usage("--sensitivity must be above zero"));
}
let threshold = SHARE_THRESHOLD / sensitivity;
let mut spans = Vec::new();
let mut examined = 0;
for display in displays {
let frame_box = self
.manifest
.displays
.iter()
.find(|entry| entry.result.display.id == *display)
.map(|entry| entry.result.display.frame);
let mut previous: Option<(Vec<u8>, u32, u32)> = None;
let mut open: Option<Open> = None;
let mut finished: Vec<Open> = Vec::new();
examined += video.scan(
*display,
from_s,
to_s,
Scale::Width(COMPARE_WIDTH),
&mut |decoded| {
if cancel
.is_some_and(|cancel| cancel.load(std::sync::atomic::Ordering::Relaxed))
{
return false;
}
let now = luma(decoded);
let (width, height) = (decoded.frame.width, decoded.frame.height);
if let Some((before, before_width, before_height)) = &previous
&& (*before_width, *before_height) == (width, height)
{
let (mut count, mut min_x, mut min_y, mut max_x, mut max_y) =
(0_usize, u32::MAX, u32::MAX, 0_u32, 0_u32);
for (index, (a, b)) in before.iter().zip(&now).enumerate() {
if (i32::from(*a) - i32::from(*b)).abs() > PIXEL_THRESHOLD {
let (x, y) = (index as u32 % width, index as u32 / width);
count += 1;
min_x = min_x.min(x);
max_x = max_x.max(x);
min_y = min_y.min(y);
max_y = max_y.max(y);
}
}
let share = count as f64 / now.len().max(1) as f64;
if share >= threshold {
match &mut open {
Some(span) if decoded.shown_s - span.end <= MERGE_GAP_S => {
span.end = decoded.shown_s;
span.frames += 1;
span.peak = span.peak.max(share);
span.bounds = (
span.bounds.0.min(min_x),
span.bounds.1.min(min_y),
span.bounds.2.max(max_x),
span.bounds.3.max(max_y),
);
}
_ => {
finished.extend(open.take());
open = Some(Open {
start: decoded.shown_s,
end: decoded.shown_s,
frames: 1,
peak: share,
bounds: (min_x, min_y, max_x, max_y),
});
}
}
}
}
previous = Some((now, width, height));
true
},
)?;
finished.extend(open.take());
let dimensions = previous
.as_ref()
.map(|(_, width, height)| (*width, *height));
for span in finished {
let (width, height) = dimensions.unwrap_or((1, 1));
let (w, h) = (f64::from(width), f64::from(height));
let bbox = [
f64::from(span.bounds.0) / w,
f64::from(span.bounds.1) / h,
f64::from(span.bounds.2 - span.bounds.0 + 1) / w,
f64::from(span.bounds.3 - span.bounds.1 + 1) / h,
];
let events = self
.events()
.iter()
.filter(|event| {
let at = self.video_s(event.recv_mach_ns);
at >= span.start - COINCIDE_S && at <= span.end + COINCIDE_S
})
.map(|event| event.seq)
.collect();
spans.push(ChangeSpan {
display: *display,
start_video_s: span.start,
end_video_s: span.end,
frames: span.frames,
peak_share: span.peak,
bbox,
bbox_points: frame_box.map(|[x, y, frame_width, frame_height]| {
[
x + bbox[0] * frame_width,
y + bbox[1] * frame_height,
bbox[2] * frame_width,
bbox[3] * frame_height,
]
}),
events,
});
}
}
spans.sort_by(|a, b| {
a.start_video_s
.total_cmp(&b.start_video_s)
.then(a.display.cmp(&b.display))
});
Ok(ChangesData {
from_video_s: from_s,
to_video_s: to_s,
displays: displays.to_vec(),
frames_examined: examined,
spans,
gaps: self.video_gaps(displays, from_s, to_s),
})
}
/// Returns the stretches of `from_s..=to_s` (seconds of video time) in which each of
/// `displays` has no video, ordered by display and then by time.
///
/// Only the manifest is read, no video is decoded. A segment of a display that recorded no
/// frames does not count as video. A stretch shorter than 0.5 s is not a gap, because a display
/// whose screen does not change produces no frames. A display that is not in the session has
/// no video anywhere, so it gets one gap that is the whole range.
///
/// # Example
///
/// ```rust
/// use komorebi_recorder::analysis::Analysis;
/// use komorebi_recorder::analysis::changes::Gap;
/// use komorebi_recorder::testkit::SessionBuilder;
///
/// // a ten second recording that has no video for display 2
/// let (session, _) = SessionBuilder::new("doc-video-gaps").length(10.0).build();
/// let analysis = Analysis::open(&session.dir)?;
///
/// assert_eq!(
/// analysis.video_gaps(&[2], 0.0, 10.0),
/// [Gap { display: 2, from_video_s: 0.0, to_video_s: 10.0 }],
/// );
/// # Ok::<(), Box<dyn std::error::Error>>(())
/// ```
///
/// A display that has video for part of the range, and a gap after it, is covered by
/// `tests/video.rs::a_display_with_no_video_for_part_of_the_range_has_a_gap`.
#[must_use]
pub fn video_gaps(&self, displays: &[u32], from_s: f64, to_s: f64) -> Vec<Gap> {
let mut gaps = Vec::new();
for display in displays {
let mut covered: Vec<(f64, f64)> = self
.manifest
.displays
.iter()
.filter(|entry| entry.result.display.id == *display && entry.result.frames > 0)
.map(|entry| {
(
entry.offset_s,
entry.end_s.unwrap_or_else(|| self.manifest.video_span().1),
)
})
.collect();
covered.sort_by(|a, b| a.0.total_cmp(&b.0));
let mut cursor = from_s;
for (start, end) in covered {
if start > cursor + MIN_GAP_S && cursor < to_s {
gaps.push(Gap {
display: *display,
from_video_s: cursor,
to_video_s: start.min(to_s),
});
}
cursor = cursor.max(end);
}
if cursor + MIN_GAP_S < to_s {
gaps.push(Gap {
display: *display,
from_video_s: cursor,
to_video_s: to_s,
});
}
}
gaps
}
}
+3
View File
@@ -12,6 +12,7 @@ pub mod blocks;
pub mod brief;
pub mod budget;
pub mod catalog;
pub mod changes;
pub mod classify;
pub mod code;
pub mod diff;
@@ -28,9 +29,11 @@ pub mod lookup;
pub mod overview;
pub mod schemas;
pub mod search;
pub mod sheet;
pub mod state;
pub mod timespec;
pub mod trace;
pub mod video;
pub mod views;
pub use facade::Analysis;
+12
View File
@@ -16,11 +16,13 @@ use super::answers::DiffData;
use super::answers::EventsData;
use super::answers::StateData;
use super::brief::BriefData;
use super::changes::ChangesData;
use super::code::CodeData;
use super::envelope::Envelope;
use super::history::HistoryData;
use super::overview::OverviewData;
use super::search::SearchData;
use super::sheet::SheetData;
use super::trace::TraceData;
use schemars::schema_for;
use serde_json::Value;
@@ -92,6 +94,16 @@ pub const ALL: &[Entry] = &[
about: "`search`: where a word appears in the recording",
schema: of::<SearchData>,
},
Entry {
name: "changes/1",
about: "`changes`: when and where the picture changed on each display",
schema: of::<ChangesData>,
},
Entry {
name: "sheet/1",
about: "`sheet`: a contact sheet image and where each frame is on it",
schema: of::<SheetData>,
},
Entry {
name: "annotation/1",
about: "`annotate add|edit`: the annotation as it now reads",
+549
View File
@@ -0,0 +1,549 @@
//! A contact sheet: frames of the recording laid out on one image, with the times, the events and
//! the bug regions marked.
//!
//! It is for someone (or an agent that can look at an image) to see what the screen did without
//! playing a video.
//!
//! Each display is its own block of frames in time order. Every frame says when it is; where
//! komorebi events fall around it they are named; frames inside a bug region the user marked have
//! a red border. Frames are decoded at the width they are drawn, so a sheet of a long recording
//! costs little.
//!
//! [`Analysis::sheet`] draws one from [`SheetOptions`] and a `SessionVideo` (the decoder of the
//! session's video, in `analysis::video`) and returns a [`SheetData`] saying where each frame is on
//! the image, so that a reader of the answer can find a frame without looking at the picture. The
//! image is a PNG written to [`SheetOptions::out`]. Decoding needs a host that can decode the video
//! (macOS and Windows; see `crate::av::supported`).
//!
//! # Layout
//!
//! From the top: a title with the session's directory name and the first and last of the moments;
//! under it, if a bug region overlaps those moments, a red line with the first such region's id and
//! its description (cut to 150 characters); then, for each display with a frame to show, a header
//! naming it and its frames in rows of [`SheetOptions::columns`]. Under each frame is its timecode
//! (red, with `BUG`, if it is inside a region) and the first two events received near it, as `#<seq>
//! <name>`, with `+<n>` for any more. An event is near a frame if it was received within half the
//! average spacing of the moments (half a second for a single moment) of the frame's moment, but
//! never less than a tenth of a second.
//!
//! The drawn width of a frame is [`SheetOptions::cell_width`], limited to between 64 and 1920
//! pixels, and the number of columns is limited to the number of moments. The image may be at most
//! 8192 pixels wide and 16384 tall; a sheet that would be larger is refused (see
//! [`Analysis::sheet`]).
//!
//! # Example
//!
//! Needs `ffmpeg` to make a test video, and a host that can decode it, so it is only compiled here;
//! the tests `a_sheet_shows_the_frame_that_was_on_screen_at_each_moment` and
//! `a_sheet_marks_the_bug_region_and_names_the_events_around_a_frame` in `tests/video.rs` run the
//! same call through the `sheet` command.
//!
//! ```rust,no_run
//! use komorebi_recorder::analysis::Analysis;
//! use komorebi_recorder::analysis::sheet::SheetOptions;
//! use komorebi_recorder::analysis::video::SessionVideo;
//! use komorebi_recorder::testkit::SessionBuilder;
//!
//! // a recording of one display, 300 frames at 30 frames per second
//! let (session, _) = SessionBuilder::new("doc-sheet").video(1, 0.0, 300, 30).build();
//! let analysis = Analysis::open(&session.dir)?;
//! let video = SessionVideo::new(&analysis.manifest, analysis.guard());
//!
//! let sheet = analysis.sheet(&video, &SheetOptions {
//! times: vec![1.0, 5.0, 9.0],
//! displays: video.displays(),
//! cell_width: 240,
//! columns: 3,
//! regions: Vec::new(),
//! out: session.path("sheets/example.png"),
//! })?;
//!
//! assert_eq!(sheet.frames.len(), 3);
//! assert!(sheet.skipped.is_empty());
//! # Ok::<(), Box<dyn std::error::Error>>(())
//! ```
use super::Analysis;
use super::envelope;
use super::video::SessionVideo;
use crate::av::Scale;
use crate::timecode;
use ab_glyph::Font;
use ab_glyph::FontRef;
use ab_glyph::PxScale;
use ab_glyph::ScaleFont;
use ab_glyph::point;
use color_eyre::eyre;
use color_eyre::eyre::WrapErr;
use schemars::JsonSchema;
use serde::Serialize;
use std::io::BufWriter;
use std::path::Path;
use std::path::PathBuf;
/// The `schema` of the `sheet` command's output: the name that [`SheetData`] has in an envelope.
pub const SCHEMA_SHEET: &str = "sheet/1";
const BACKGROUND: [u8; 3] = [24, 24, 28];
const TEXT: [u8; 3] = [225, 225, 230];
const DIM: [u8; 3] = [140, 140, 150];
const EVENT: [u8; 3] = [240, 190, 70];
const BUG: [u8; 3] = [230, 70, 60];
const MARGIN: u32 = 10;
const LABEL_HEIGHT: u32 = 44;
const HEADER_HEIGHT: u32 = 26;
const TITLE_HEIGHT: u32 = 44;
const MAX_WIDTH: u32 = 8192;
const MAX_HEIGHT: u32 = 16384;
/// A bug region to mark on the sheet.
#[derive(Debug, Clone)]
pub struct Region {
/// The annotation's id, written on the sheet and in [`SheetFrame::bugs`].
pub id: String,
/// Where the region starts, in seconds of video time.
pub start_s: f64,
/// Where the region ends, in seconds of video time. The `sheet` command gives an open-ended
/// bug the end of the recording.
pub end_s: f64,
/// What the user said is wrong, written under the title of the sheet.
pub text: String,
}
/// What to put on a sheet, and where to write it.
#[derive(Debug, Clone)]
pub struct SheetOptions {
/// The moments to show, in session video time. They are drawn in this order, so a caller that
/// wants them in time order sorts them. Must not be empty.
pub times: Vec<f64>,
/// The displays to show, each as a block of its own. Must not be empty; a display listed that
/// has no video at any of the moments is left out.
pub displays: Vec<u32>,
/// How wide each frame is drawn, in pixels; limited to between 64 and 1920.
pub cell_width: u32,
/// How many frames to a row; limited to between 1 and the number of moments.
pub columns: usize,
/// The bug regions to mark: a frame whose moment is inside one has a red border.
pub regions: Vec<Region>,
/// Where to write the PNG. Its parent directories are created, and a file already there is
/// replaced.
pub out: PathBuf,
}
/// One frame of a sheet and where it is on the image.
#[derive(Debug, Clone, PartialEq, Serialize, JsonSchema)]
pub struct SheetFrame {
/// The display the frame is of.
pub display: u32,
/// The moment that was asked for, in seconds of video time.
pub requested_video_s: f64,
/// When the frame shown was actually put on screen, in seconds of video time. It is the picture
/// that was on screen at the requested moment, so it is never later than it, and can be
/// earlier: a still screen produces no new frames.
pub shown_video_s: f64,
/// Where it is on the image: pixels from the left edge to its left edge.
pub x: u32,
/// Where it is on the image: pixels from the top edge to its top edge.
pub y: u32,
/// How wide it is drawn, in pixels.
pub width: u32,
/// How tall it is drawn, in pixels.
pub height: u32,
/// The sequence numbers of the events received near it.
pub events: Vec<u64>,
/// The ids of the bug regions it is inside.
pub bugs: Vec<String>,
}
/// A moment of a display that the sheet has no frame for.
#[derive(Debug, Clone, PartialEq, Serialize, JsonSchema)]
pub struct Skipped {
/// The display that was asked for.
pub display: u32,
/// The moment that was asked for, in seconds of video time.
pub video_s: f64,
/// Why there is no frame.
pub reason: String,
}
/// The answer to `sheet`: the image that was written and where each frame is on it.
#[derive(Debug, Clone, PartialEq, Serialize, JsonSchema)]
pub struct SheetData {
/// Where the PNG was written.
pub path: String,
/// The width of the image, in pixels.
pub width: u32,
/// The height of the image, in pixels.
pub height: u32,
/// The frames on the image, a display at a time and in the order of the moments within it.
pub frames: Vec<SheetFrame>,
/// Moments a display had no video.
pub skipped: Vec<Skipped>,
}
struct Canvas {
width: u32,
height: u32,
rgba: Vec<u8>,
}
impl Canvas {
fn new(width: u32, height: u32) -> Self {
let mut rgba = Vec::with_capacity(width as usize * height as usize * 4);
for _ in 0..width as usize * height as usize {
rgba.extend_from_slice(&[BACKGROUND[0], BACKGROUND[1], BACKGROUND[2], 255]);
}
Self {
width,
height,
rgba,
}
}
fn put(&mut self, x: i64, y: i64, color: [u8; 3], alpha: f32) {
if x < 0 || y < 0 || x >= i64::from(self.width) || y >= i64::from(self.height) {
return;
}
let at = (y as usize * self.width as usize + x as usize) * 4;
for (channel, wanted) in color.iter().enumerate() {
let below = f32::from(self.rgba[at + channel]);
self.rgba[at + channel] = (below + (f32::from(*wanted) - below) * alpha).round() as u8;
}
}
fn rectangle(&mut self, x: u32, y: u32, width: u32, height: u32, color: [u8; 3]) {
for row in y..y + height {
for column in x..x + width {
self.put(i64::from(column), i64::from(row), color, 1.0);
}
}
}
fn border(&mut self, x: u32, y: u32, width: u32, height: u32, thickness: u32, color: [u8; 3]) {
self.rectangle(x, y, width, thickness, color);
self.rectangle(x, y + height - thickness, width, thickness, color);
self.rectangle(x, y, thickness, height, color);
self.rectangle(x + width - thickness, y, thickness, height, color);
}
/// Copy a BGRA frame in at `x`, `y`.
fn blit(&mut self, x: u32, y: u32, width: u32, height: u32, bgra: &[u8]) {
for row in 0..height {
for column in 0..width {
let from = (row as usize * width as usize + column as usize) * 4;
if from + 2 >= bgra.len() {
return;
}
self.put(
i64::from(x + column),
i64::from(y + row),
[bgra[from + 2], bgra[from + 1], bgra[from]],
1.0,
);
}
}
}
fn text(&mut self, font: &FontRef, size: f32, x: u32, y: u32, text: &str, color: [u8; 3]) {
let scaled = font.as_scaled(PxScale::from(size));
let baseline = y as f32 + scaled.ascent();
let mut caret = x as f32;
for character in text.chars() {
let id = scaled.glyph_id(character);
let glyph = id.with_scale_and_position(PxScale::from(size), point(caret, baseline));
caret += scaled.h_advance(id);
if let Some(outlined) = font.outline_glyph(glyph) {
let bounds = outlined.px_bounds();
outlined.draw(|gx, gy, coverage| {
self.put(
i64::from(bounds.min.x as i32) + i64::from(gx),
i64::from(bounds.min.y as i32) + i64::from(gy),
color,
coverage,
);
});
}
}
}
fn save(&self, path: &Path) -> eyre::Result<()> {
if let Some(parent) = path.parent() {
std::fs::create_dir_all(parent)
.wrap_err_with(|| format!("could not create {}", parent.display()))?;
}
let file = std::fs::File::create(path)
.wrap_err_with(|| format!("could not create {}", path.display()))?;
let mut encoder = png::Encoder::new(BufWriter::new(file), self.width, self.height);
encoder.set_color(png::ColorType::Rgba);
encoder.set_depth(png::BitDepth::Eight);
let mut writer = encoder.write_header()?;
writer.write_image_data(&self.rgba)?;
writer.finish()?;
Ok(())
}
}
fn truncate(text: &str, limit: usize) -> String {
let single: String = text.split_whitespace().collect::<Vec<_>>().join(" ");
if single.chars().count() <= limit {
single
} else {
let mut cut: String = single.chars().take(limit.saturating_sub(1)).collect();
cut.push('…');
cut
}
}
impl Analysis {
/// Draws a contact sheet of the moments in `options` and writes it as a PNG.
///
/// Each frame is the one that was on screen at its moment, decoded at the drawn width. A moment
/// at which a display had no video (it was unplugged, asleep, or not yet recorded) is skipped
/// and listed in [`SheetData::skipped`] rather than failing the sheet. See the module docs for
/// the layout.
///
/// # Errors
///
/// - a usage error if `options.times` is empty, or if the sheet would be wider than 8192 or
/// taller than 16384 pixels (the message says how big it would be; show fewer frames or draw
/// them narrower);
/// - an unusable error if `options.displays` is empty, which is what the commands pass for a
/// session that recorded no video;
/// - a not-found error if none of the moments has video on any of the displays;
/// - an error if a video file cannot be opened or decoded (including on a host that cannot
/// decode video), if the output directory or file cannot be created, or if the PNG cannot be
/// written.
///
/// # Example
///
/// See the [module docs](self).
pub fn sheet(&self, video: &SessionVideo, options: &SheetOptions) -> eyre::Result<SheetData> {
if options.times.is_empty() {
return Err(envelope::usage("there are no moments to show"));
}
if options.displays.is_empty() {
return Err(envelope::unusable("the session recorded no video"));
}
let font = FontRef::try_from_slice(epaint_default_fonts::HACK_REGULAR)
.map_err(|_| eyre::Report::msg("the built-in font could not be read"))?;
let columns = options.columns.clamp(1, options.times.len());
let cell = options.cell_width.clamp(64, 1920);
// decode every frame first, so the layout knows each one's height
struct Cell {
requested: f64,
shown: f64,
width: u32,
height: u32,
bgra: Vec<u8>,
}
let mut blocks: Vec<(u32, Vec<Cell>)> = Vec::new();
let mut skipped = Vec::new();
for display in &options.displays {
let mut cells = Vec::new();
for time in &options.times {
match video.frame_at(*display, *time, Scale::Width(cell))? {
Some(decoded) => cells.push(Cell {
requested: *time,
shown: decoded.shown_s,
width: decoded.frame.width,
height: decoded.frame.height,
bgra: decoded.frame.bgra,
}),
None => skipped.push(Skipped {
display: *display,
video_s: *time,
reason:
"the display had no video then (unplugged, asleep, or not yet recorded)"
.to_string(),
}),
}
}
if !cells.is_empty() {
blocks.push((*display, cells));
}
}
if blocks.is_empty() {
return Err(envelope::not_found(
"none of those moments has video on any display",
));
}
let width = MARGIN + columns as u32 * (cell + MARGIN);
let mut height = TITLE_HEIGHT + MARGIN;
for (_, cells) in &blocks {
height += HEADER_HEIGHT;
for row in cells.chunks(columns) {
height +=
row.iter().map(|cell| cell.height).max().unwrap_or(0) + LABEL_HEIGHT + MARGIN;
}
}
if width > MAX_WIDTH || height > MAX_HEIGHT {
return Err(envelope::usage(format!(
"a sheet of {} frames at width {cell} would be {width}x{height}: show fewer frames \
or draw them narrower",
options.times.len()
)));
}
let mut canvas = Canvas::new(width, height);
let first = options.times.iter().copied().fold(f64::INFINITY, f64::min);
let last = options
.times
.iter()
.copied()
.fold(f64::NEG_INFINITY, f64::max);
let session_name = self
.session
.dir
.file_name()
.map_or_else(String::new, |name| name.to_string_lossy().into_owned());
canvas.text(
&font,
15.0,
MARGIN,
6,
&format!(
"{session_name} {} - {}",
timecode::format(first),
timecode::format(last)
),
TEXT,
);
if let Some(region) = options
.regions
.iter()
.find(|region| region.end_s >= first && region.start_s <= last)
{
canvas.text(
&font,
12.0,
MARGIN,
26,
&format!("BUG {}: {}", region.id, truncate(&region.text, 150)),
BUG,
);
}
// spread of the moments, to know how near an event has to be to belong to a frame
let step = if options.times.len() > 1 {
(last - first) / (options.times.len() - 1) as f64
} else {
1.0
};
let reach = (step / 2.0).max(0.1);
let mut frames = Vec::new();
let mut y = TITLE_HEIGHT + MARGIN;
for (display, cells) in &blocks {
canvas.text(
&font,
14.0,
MARGIN,
y + 4,
&format!("display {display}"),
DIM,
);
y += HEADER_HEIGHT;
for row in cells.chunks(columns) {
let row_height = row.iter().map(|cell| cell.height).max().unwrap_or(0);
for (column, cell_frame) in row.iter().enumerate() {
let x = MARGIN + column as u32 * (cell + MARGIN);
canvas.blit(x, y, cell_frame.width, cell_frame.height, &cell_frame.bgra);
let bugs: Vec<String> = options
.regions
.iter()
.filter(|region| {
cell_frame.requested >= region.start_s - 1e-6
&& cell_frame.requested <= region.end_s + 1e-6
})
.map(|region| region.id.clone())
.collect();
if !bugs.is_empty() {
canvas.border(x, y, cell_frame.width, cell_frame.height, 3, BUG);
}
let events: Vec<u64> = self
.events()
.iter()
.filter(|event| {
(self.video_s(event.recv_mach_ns) - cell_frame.requested).abs() <= reach
})
.map(|event| event.seq)
.collect();
let label_y = y + row_height + 3;
canvas.text(
&font,
13.0,
x,
label_y,
&timecode::format(cell_frame.requested),
if bugs.is_empty() { TEXT } else { BUG },
);
if !bugs.is_empty() {
canvas.text(&font, 12.0, x + 92, label_y, "BUG", BUG);
}
let named: Vec<String> = events
.iter()
.take(2)
.filter_map(|seq| {
let index = self.event_index(*seq)?;
let event = &self.events()[index];
Some(format!("#{seq} {}", event.info.name))
})
.collect();
if !named.is_empty() {
let more = events.len().saturating_sub(2);
canvas.text(
&font,
11.0,
x,
label_y + 16,
&truncate(
&format!(
"{}{}",
named.join(" "),
if more > 0 {
format!(" +{more}")
} else {
String::new()
}
),
(cell / 7) as usize,
),
EVENT,
);
}
frames.push(SheetFrame {
display: *display,
requested_video_s: cell_frame.requested,
shown_video_s: cell_frame.shown,
x,
y,
width: cell_frame.width,
height: cell_frame.height,
events,
bugs,
});
}
y += row_height + LABEL_HEIGHT + MARGIN;
}
}
canvas.save(&options.out)?;
Ok(SheetData {
path: options.out.display().to_string(),
width,
height,
frames,
skipped,
})
}
}
+354
View File
@@ -0,0 +1,354 @@
//! The recorded video, addressed by session time and display.
//!
//! A display's video is one file per segment (a display that was unplugged or slept and came
//! back starts a new one), and a file's own time starts at its first frame. This maps a moment of
//! session video time on a display to the file and file time that has it, decodes frames, and says
//! when there is no video to show (the display was absent).
//!
//! [`SessionVideo`] is what the `changes` and `sheet` commands and the viewer's playback ask for
//! frames. It uses the decoder of `crate::av` (`AVFoundation` on macOS, Media Foundation on Windows),
//! so a host with neither cannot open a file; see `crate::av::supported`.
//!
//! # Times
//!
//! Every time here is *session video time*: seconds from the first frame of any display
//! ([`crate::session::Manifest::video_t0`]). A segment's file has its own time, which starts at its
//! first frame, and the manifest says how far into session time that is ([`DisplayEntry::offset_s`]):
//! session time `t` is at `t - offset_s` in the file. A moment is covered by a segment of a display
//! if it falls between the segment's start and end, with 50 ms of slack on each side, since
//! segment boundaries are sampled and not exact; where two segments of a display both cover a
//! moment, the later one is used ([`crate::session::Manifest::segment_at`]). A moment covered by
//! none is a gap (the display was unplugged, asleep, or not yet recorded) and the methods say so
//! with `None`, never with an invented frame.
//!
//! | To get | Use | Crosses segments |
//! |---|---|---|
//! | the frame on screen at one moment | [`SessionVideo::frame_at`] | no |
//! | the frames from one moment on, one at a time | [`SessionVideo::open_at`] | no |
//! | every frame of a range | [`SessionVideo::scan`] | yes |
//! | which displays have video, and whether one does at a moment | [`SessionVideo::displays`], [`SessionVideo::has_video_at`] | |
//!
//! # Example
//!
//! Needs `ffmpeg` to make a test video, and a host that can decode it, so it is only compiled here;
//! the tests of `tests/video.rs` (`a_sheet_shows_the_frame_that_was_on_screen_at_each_moment`,
//! `a_moment_a_display_had_no_video_is_skipped_not_invented`) run these calls through the `sheet`
//! command.
//!
//! ```rust,no_run
//! use komorebi_recorder::analysis::Analysis;
//! use komorebi_recorder::analysis::video::SessionVideo;
//! use komorebi_recorder::av::Scale;
//! use komorebi_recorder::testkit::SessionBuilder;
//!
//! // display 1 has video for 3 seconds from the start; display 2 only from 1 s to 3 s
//! let (session, _) = SessionBuilder::new("doc-video")
//! .video(1, 0.0, 90, 30)
//! .video(2, 1.0, 60, 30)
//! .build();
//! let analysis = Analysis::open(&session.dir)?;
//! let video = SessionVideo::new(&analysis.manifest, analysis.guard());
//!
//! assert_eq!(video.displays(), [1, 2]);
//! assert!(video.has_video_at(1, 0.5));
//! assert!(!video.has_video_at(2, 0.5));
//!
//! // the frame on screen at 1.5 s, at most 128 pixels wide; never shown later than asked
//! let decoded = video.frame_at(1, 1.5, Scale::Width(128))?.expect("display 1 had video");
//! assert!(decoded.shown_s <= 1.5);
//!
//! // display 2 had none at 0.5 s
//! assert!(video.frame_at(2, 0.5, Scale::Width(128))?.is_none());
//! # Ok::<(), Box<dyn std::error::Error>>(())
//! ```
use super::guard::SessionPath;
use crate::av::Asset;
use crate::av::Frame;
use crate::av::Scale;
use crate::session::DisplayEntry;
use crate::session::Manifest;
use color_eyre::eyre;
use std::cell::RefCell;
use std::collections::HashMap;
use std::path::PathBuf;
use std::rc::Rc;
/// A decoded frame and when it was actually on screen.
pub struct DecodedFrame {
/// The display the frame is of.
pub display: u32,
/// The segment of the display's video it came from.
pub segment: u32,
/// When the frame was shown, in session video time. Never later than what was asked for.
///
/// A screen that holds still produces no new frames, so the frame that was on screen at a
/// moment can have been put there some time before it.
pub shown_s: f64,
/// The decoded picture, scaled as asked.
pub frame: Frame,
}
/// A started decode of one segment of one display. Frames come in order, each with the session
/// video time it was on screen.
///
/// Made by [`SessionVideo::open_at`]. A stream stays within its segment: it does not continue into
/// the next one.
pub struct Stream {
reader: crate::av::Reader,
offset_s: f64,
/// The segment of the display's video it reads.
pub segment: u32,
/// Session video time at which the segment ends.
pub end_s: f64,
}
impl Stream {
/// Returns the next frame and when it was shown, in session video time, or `None` at the end of
/// the segment (or of the range the stream was opened with).
///
/// # Errors
///
/// Returns an error if the decoder fails.
pub fn next_frame(&mut self) -> eyre::Result<Option<(f64, Frame)>> {
Ok(self
.reader
.next_frame()?
.map(|frame| (self.offset_s + frame.pts_s, frame)))
}
}
/// The video of a session. Not `Send`: decoding objects belong to the thread that made them.
///
/// A file is opened the first time a frame of it is asked for and kept open for the life of the
/// `SessionVideo`, so asking for many frames of the same segment opens it once. It borrows the
/// session's [`Manifest`] and [`SessionPath`] (`Analysis::manifest` and `Analysis::guard`), and the
/// guard is what keeps a file named in the manifest from leading outside the session.
///
/// See the [module docs](self) for how session time maps to files and an example.
///
/// # Example: it stays on the thread that made it
///
/// The compiler refuses to send one to another thread:
///
/// ```rust,compile_fail,E0277
/// use komorebi_recorder::analysis::video::SessionVideo;
///
/// fn send_to_another_thread<T: Send>() {}
/// send_to_another_thread::<SessionVideo<'static>>();
/// ```
///
/// A thread that decodes makes its own, from a manifest and a guard that can be shared (an
/// [`Analysis`](super::Analysis) is `Send` and `Sync`):
///
/// ```rust
/// use komorebi_recorder::analysis::Analysis;
/// use komorebi_recorder::analysis::video::SessionVideo;
/// use komorebi_recorder::testkit::SessionBuilder;
/// use std::sync::Arc;
///
/// let (session, _) = SessionBuilder::new("doc-video-threads").build();
/// let analysis = Arc::new(Analysis::open(&session.dir)?);
///
/// let shared = Arc::clone(&analysis);
/// let displays = std::thread::spawn(move || {
/// SessionVideo::new(&shared.manifest, shared.guard()).displays()
/// })
/// .join()
/// .expect("the thread finished");
/// assert!(displays.is_empty(), "a session recorded without video has no displays with video");
/// # Ok::<(), Box<dyn std::error::Error>>(())
/// ```
pub struct SessionVideo<'a> {
manifest: &'a Manifest,
guard: &'a SessionPath,
assets: RefCell<HashMap<PathBuf, Rc<Asset>>>,
}
impl<'a> SessionVideo<'a> {
/// Returns the video of the session described by `manifest`, resolving its files through
/// `guard`. Nothing is opened until a frame is asked for.
#[must_use]
pub fn new(manifest: &'a Manifest, guard: &'a SessionPath) -> Self {
Self {
manifest,
guard,
assets: RefCell::new(HashMap::new()),
}
}
/// Returns the ids of the displays that have any video (a segment with at least one frame), in
/// ascending order.
#[must_use]
pub fn displays(&self) -> Vec<u32> {
let mut ids: Vec<u32> = self
.manifest
.displays
.iter()
.filter(|entry| entry.result.frames > 0)
.map(|entry| entry.result.display.id)
.collect();
ids.sort_unstable();
ids.dedup();
ids
}
fn asset(&self, entry: &DisplayEntry) -> eyre::Result<Rc<Asset>> {
let path = self.guard.resolve(&entry.result.file)?;
if let Some(found) = self.assets.borrow().get(&path) {
return Ok(found.clone());
}
let asset = Rc::new(Asset::open(&path)?);
self.assets.borrow_mut().insert(path, asset.clone());
Ok(asset)
}
/// Returns the frame of `display` on screen at session video time `video_s`, decoded at
/// `scale`, or `None` if the display had no video then.
///
/// The frame's [`DecodedFrame::shown_s`] is the time it was put on screen, which is never later
/// than `video_s` and can be earlier. A time at or past the end of the file asks for its last
/// frame, not for something beyond it. `None` is also returned if the decoder produces no frame
/// for the moment. The display's video is looked up as [`SessionVideo`]'s module docs describe,
/// so a moment in the slack at the edge of a segment is answered with that segment's nearest
/// frame.
///
/// # Errors
///
/// Returns an error if the segment's file leads outside the session (a usage error), cannot be
/// opened (it is missing or damaged, or the host cannot decode video at all), or cannot be
/// decoded.
pub fn frame_at(
&self,
display: u32,
video_s: f64,
scale: Scale,
) -> eyre::Result<Option<DecodedFrame>> {
let Some(entry) = self.manifest.segment_at(display, video_s) else {
return Ok(None);
};
let asset = self.asset(entry)?;
// the last moment of a file is the last frame, not past it
let file_s = (video_s - entry.offset_s)
.max(0.0)
.min((asset.duration_s() - 0.002).max(0.0));
let mut reader = asset.start(file_s, Some(file_s + 0.5), scale)?;
Ok(reader.next_frame()?.map(|frame| DecodedFrame {
display,
segment: entry.result.segment,
shown_s: entry.offset_s + frame.pts_s,
frame,
}))
}
/// Returns `true` if and only if `display` has video at session video time `video_s`: a segment
/// of it covers the moment (see the [module docs](self)). It does not open any file.
#[must_use]
pub fn has_video_at(&self, display: u32, video_s: f64) -> bool {
self.manifest.segment_at(display, video_s).is_some()
}
/// Starts decoding the segment of `display` that has video at `video_s`, beginning with the
/// frame on screen then and running to `to_s` (or the end of the segment). `None` if the
/// display had no video then.
///
/// The stream ends at the end of the segment even if `to_s` is later: use
/// [`SessionVideo::scan`] to read across segments. A `to_s` before `video_s` is read as
/// `video_s`.
///
/// # Errors
///
/// Returns an error under the same conditions as [`SessionVideo::frame_at`].
pub fn open_at(
&self,
display: u32,
video_s: f64,
to_s: Option<f64>,
scale: Scale,
) -> eyre::Result<Option<Stream>> {
let Some(entry) = self.manifest.segment_at(display, video_s) else {
return Ok(None);
};
let asset = self.asset(entry)?;
let start = (video_s - entry.offset_s)
.max(0.0)
.min((asset.duration_s() - 0.002).max(0.0));
let stop = to_s.map(|to| (to - entry.offset_s).max(start));
Ok(Some(Stream {
reader: asset.start(start, stop, scale)?,
offset_s: entry.offset_s,
segment: entry.result.segment,
end_s: entry.end_s.unwrap_or_else(|| self.manifest.video_span().1),
}))
}
/// Decodes every frame of `display` from `from_s` to `to_s`, across segments, in order, calling
/// `visit` with each until it returns `false`. Returns how many frames were visited, counting
/// the one `visit` stopped on.
///
/// A frame shown before `from_s` is visited only if it was shown less than half a second before
/// it: the decoder starts at the frame that was on screen at `from_s`, which can be earlier than
/// it, and that frame is not visited if it is older than that. No frame shown after `to_s` is
/// visited. A gap between segments is not visited: there is nothing to decode there.
///
/// # Errors
///
/// Returns an error under the same conditions as [`SessionVideo::frame_at`], for the first
/// segment that fails. Frames visited before it stay visited.
pub fn scan(
&self,
display: u32,
from_s: f64,
to_s: f64,
scale: Scale,
visit: &mut dyn FnMut(&DecodedFrame) -> bool,
) -> eyre::Result<usize> {
let mut segments: Vec<&DisplayEntry> = self
.manifest
.displays
.iter()
.filter(|entry| entry.result.display.id == display && entry.result.frames > 0)
.collect();
segments.sort_by(|a, b| a.offset_s.total_cmp(&b.offset_s));
let (_, session_end) = self.manifest.video_span();
let mut visited = 0;
for entry in segments {
let end = entry.end_s.unwrap_or(session_end);
if end < from_s || entry.offset_s > to_s {
continue;
}
let asset = self.asset(entry)?;
let start = (from_s - entry.offset_s).max(0.0);
let stop = (to_s - entry.offset_s).min(asset.duration_s());
let mut reader = asset.start(start, Some(stop), scale)?;
while let Some(frame) = reader.next_frame()? {
let shown_s = entry.offset_s + frame.pts_s;
// the decoder starts at the frame on screen at `from_s`, which can be before it
if shown_s + 1e-6 < from_s && shown_s + 0.5 < from_s {
continue;
}
if shown_s > to_s {
break;
}
visited += 1;
let decoded = DecodedFrame {
display,
segment: entry.result.segment,
shown_s,
frame,
};
if !visit(&decoded) {
return Ok(visited);
}
}
}
Ok(visited)
}
}
+11
View File
@@ -11,6 +11,7 @@ mod doctor;
mod investigate;
mod query;
mod record;
mod video;
use crate::analysis::envelope::unusable;
use crate::analysis::envelope::with_hint;
@@ -43,6 +44,8 @@ pub use query::LocateArgs;
pub use query::RangeArgs;
pub use query::SummaryArgs;
pub use record::RecordArgs;
pub use video::ChangesArgs;
pub use video::SheetArgs;
/// The command line of `komorebi-recorder`.
#[derive(Parser)]
@@ -105,6 +108,10 @@ pub enum Command {
Window(WindowArgs),
/// One event end to end: what its handler logged, what changed, and who changed it
Trace(TraceArgs),
/// When and where on each display the picture changed
Changes(ChangesArgs),
/// A contact sheet image of frames, with the times, events and bug region marked
Sheet(SheetArgs),
/// Which line of komorebi's source wrote a line of its log, with the code around it
Code(CodeArgs),
/// Add, edit or delete a session's bug regions, notes and an agent's findings
@@ -171,6 +178,8 @@ impl Command {
Self::Code(args) => args.json,
Self::Trace(args) => args.json,
Self::Window(args) => args.json,
Self::Changes(args) => args.json,
Self::Sheet(args) => args.json,
Self::Annotate(args) => args.json(),
Self::Annotations(args) => args.json,
Self::Doctor(args) => args.json,
@@ -229,6 +238,8 @@ pub fn run(command: Command) -> eyre::Result<()> {
Command::Code(args) => investigate::code(&args),
Command::Trace(args) => investigate::trace(&args),
Command::Window(args) => investigate::window(&args),
Command::Changes(args) => video::changes(&args),
Command::Sheet(args) => video::sheet(&args),
Command::Annotate(args) => annotate::annotate(&args),
Command::Annotations(args) => annotate::list(&args),
Command::Reindex(args) => reindex(&args),
+235
View File
@@ -0,0 +1,235 @@
//! `changes` and `sheet`: what the screen did.
//!
//! | Command | Asks | Answer, with `--json` |
//! |---|---|---|
//! | `changes` | [`Analysis::changes`](crate::analysis::Analysis::changes) | `changes/1` |
//! | `sheet` | [`Analysis::sheet`] | `sheet/1` |
//!
//! Both work on a range of video time, given as `--from` and `--to` (any time `state --at` takes;
//! the default is the whole video) or, instead of those, as `--bug <id>`: a bug region from three
//! seconds before it starts to its end. Both work on the displays that have video (`--display`, repeatable, to pick),
//! and both refuse a session that recorded no video. `sheet` also takes `--at` for exact moments, or
//! `--count` moments evenly spaced across the range, and writes a PNG (by default to
//! `<session>/sheets/<range>.png`); see [`crate::analysis::sheet`] for the layout.
//!
//! Needs a host that can decode the video (macOS and Windows; see [`crate::av::supported`]).
//! Covered by `tests/video.rs`, which runs the commands through the real binary on a session with a
//! test video (it skips with a message when `ffmpeg` is not there to make one).
use super::print_json;
use crate::analysis::Analysis;
use crate::analysis::changes::SCHEMA_CHANGES;
use crate::analysis::envelope;
use crate::analysis::envelope::Envelope;
use crate::analysis::sheet::Region;
use crate::analysis::sheet::SCHEMA_SHEET;
use crate::analysis::sheet::SheetOptions;
use crate::analysis::video::SessionVideo;
use crate::annotations;
use crate::report;
use clap::Args;
use color_eyre::eyre;
use std::path::PathBuf;
/// Seconds shown before a bug region when a sheet is made of one.
const BUG_PRE_ROLL_S: f64 = 3.0;
/// The arguments of `changes`.
#[derive(Args, Debug)]
pub struct ChangesArgs {
/// Session directory
pub session: PathBuf,
/// From this moment [default: the start of the video]
#[arg(long, value_name = "TIME")]
pub from: Option<String>,
/// To this moment [default: the end of the video]
#[arg(long, value_name = "TIME")]
pub to: Option<String>,
/// Instead: a bug region, from a few seconds before it to its end
#[arg(long, value_name = "ID", conflicts_with_all = ["from", "to"])]
pub bug: Option<String>,
/// Only this display (repeatable) [default: every display with video]
#[arg(long = "display", value_name = "ID")]
pub displays: Vec<u32>,
/// Above 1 notices smaller changes, below 1 only larger ones
#[arg(long, default_value_t = 1.0)]
pub sensitivity: f64,
/// Machine-readable JSON output
#[arg(long)]
pub json: bool,
}
/// The arguments of `sheet`.
#[derive(Args, Debug)]
pub struct SheetArgs {
/// Session directory
pub session: PathBuf,
/// From this moment [default: the start of the video]
#[arg(long, value_name = "TIME")]
pub from: Option<String>,
/// To this moment [default: the end of the video]
#[arg(long, value_name = "TIME")]
pub to: Option<String>,
/// Instead: a bug region, from a few seconds before it to its end
#[arg(long, value_name = "ID", conflicts_with_all = ["from", "to"])]
pub bug: Option<String>,
/// Show exactly these moments instead of evenly spaced ones (repeatable)
#[arg(long = "at", value_name = "TIME")]
pub at: Vec<String>,
/// How many frames, evenly spaced
#[arg(long, default_value_t = 12)]
pub count: usize,
/// Only this display (repeatable) [default: every display with video]
#[arg(long = "display", value_name = "ID")]
pub displays: Vec<u32>,
/// How wide each frame is drawn, in pixels
#[arg(long, default_value_t = 480)]
pub width: u32,
/// Frames per row
#[arg(long, default_value_t = 4)]
pub columns: usize,
/// Where to write the PNG [default: in the session's sheets/ directory, named by the range]
#[arg(long, value_name = "FILE")]
pub out: Option<PathBuf>,
/// Machine-readable JSON output
#[arg(long)]
pub json: bool,
}
/// The video-time range to work on: a bug region, or `--from`/`--to`, or everything.
fn range(
analysis: &Analysis,
from: Option<&str>,
to: Option<&str>,
bug: Option<&str>,
) -> eyre::Result<(f64, f64)> {
if let Some(id) = bug {
let loaded = annotations::load(&analysis.session.dir)?;
let found = loaded
.get(id)
.ok_or_else(|| envelope::not_found(format!("there is no annotation {id}")))?;
let end = found.end_or_session_end(&analysis.manifest);
return Ok((
(analysis.video_s(found.start_mach_ns) - BUG_PRE_ROLL_S).max(0.0),
analysis.video_s(end),
));
}
let resolve = |text: Option<&str>, default: f64| -> eyre::Result<f64> {
text.map_or(Ok(default), |text| {
analysis.resolve(text).map(|resolved| resolved.video_s)
})
};
let (_, end) = analysis.manifest.video_span();
let (from, to) = (resolve(from, 0.0)?, resolve(to, end)?);
if to < from {
return Err(envelope::usage("the end of the range is before its start"));
}
Ok((from, to))
}
fn displays(video: &SessionVideo, wanted: &[u32]) -> eyre::Result<Vec<u32>> {
let available = video.displays();
if available.is_empty() {
return Err(envelope::unusable("the session recorded no video"));
}
if wanted.is_empty() {
return Ok(available);
}
for display in wanted {
if !available.contains(display) {
return Err(envelope::not_found(format!(
"display {display} has no video; displays with video: {available:?}"
)));
}
}
Ok(wanted.to_vec())
}
pub(super) fn changes(args: &ChangesArgs) -> eyre::Result<()> {
let analysis = Analysis::open(&args.session)?;
let video = SessionVideo::new(&analysis.manifest, analysis.guard());
let (from, to) = range(
&analysis,
args.from.as_deref(),
args.to.as_deref(),
args.bug.as_deref(),
)?;
let shown = displays(&video, &args.displays)?;
let data = analysis.changes(&video, &shown, from, to, args.sensitivity)?;
if args.json {
return print_json(&Envelope::ok(SCHEMA_CHANGES, data).warnings(analysis.warnings()));
}
print!("{}", report::changes_text(&data));
Ok(())
}
pub(super) fn sheet(args: &SheetArgs) -> eyre::Result<()> {
let analysis = Analysis::open(&args.session)?;
let video = SessionVideo::new(&analysis.manifest, analysis.guard());
let (from, to) = range(
&analysis,
args.from.as_deref(),
args.to.as_deref(),
args.bug.as_deref(),
)?;
let shown = displays(&video, &args.displays)?;
let times: Vec<f64> = if args.at.is_empty() {
let count = args.count.max(1);
if count == 1 {
vec![from]
} else {
(0..count)
.map(|index| from + (to - from) * index as f64 / (count - 1) as f64)
.collect()
}
} else {
args.at
.iter()
.map(|text| analysis.resolve(text).map(|resolved| resolved.video_s))
.collect::<eyre::Result<_>>()?
};
let regions: Vec<Region> = annotations::load(&analysis.session.dir)?
.bugs()
.map(|bug| Region {
id: bug.id.clone(),
start_s: analysis.video_s(bug.start_mach_ns),
end_s: analysis.video_s(bug.end_or_session_end(&analysis.manifest)),
text: bug.text.clone(),
})
.collect();
let out = args.out.clone().unwrap_or_else(|| {
analysis.session.path(format!(
"sheets/{}-{}.png",
crate::timecode::format(times.iter().copied().fold(f64::INFINITY, f64::min))
.replace(':', "-"),
crate::timecode::format(times.iter().copied().fold(f64::NEG_INFINITY, f64::max))
.replace(':', "-"),
))
});
let data = analysis.sheet(
&video,
&SheetOptions {
times,
displays: shown,
cell_width: args.width,
columns: args.columns,
regions,
out,
},
)?;
if args.json {
return print_json(&Envelope::ok(SCHEMA_SHEET, data).warnings(analysis.warnings()));
}
print!("{}", report::sheet_text(&data));
Ok(())
}
+143
View File
@@ -31,6 +31,7 @@ use crate::analysis::answers::EventsData;
use crate::analysis::answers::StateData;
use crate::analysis::brief::BriefData;
use crate::analysis::brief::RegionBrief;
use crate::analysis::changes::ChangesData;
use crate::analysis::code::CodeData;
use crate::analysis::history::HistoryData;
use crate::analysis::invariants::Severity;
@@ -38,6 +39,7 @@ use crate::analysis::invariants::Violation;
use crate::analysis::overview::Finding;
use crate::analysis::overview::OverviewData;
use crate::analysis::search::SearchData;
use crate::analysis::sheet::SheetData;
use crate::analysis::state::Container;
use crate::analysis::state::Window;
use crate::analysis::trace::TraceData;
@@ -1554,6 +1556,147 @@ pub fn search_text(data: &SearchData) -> String {
out
}
/// Renders the answer to `changes`: how many frames were compared, then a line for each stretch of
/// change, and the stretches in which a display had no video.
///
/// A stretch of change has its display, times and length, the most of the screen that changed
/// between two frames, the box around what changed as percentages of the screen, and the events
/// received during it.
///
/// # Example
///
/// ```rust
/// use komorebi_recorder::analysis::changes::{ChangeSpan, ChangesData, Gap};
/// use komorebi_recorder::report::changes_text;
///
/// let data = ChangesData {
/// from_video_s: 0.0,
/// to_video_s: 4.0,
/// displays: vec![1],
/// frames_examined: 120,
/// spans: vec![ChangeSpan {
/// display: 1,
/// start_video_s: 1.0,
/// end_video_s: 1.5,
/// frames: 15,
/// peak_share: 0.25,
/// bbox: [0.5, 0.0, 0.5, 1.0],
/// bbox_points: None,
/// events: vec![3],
/// }],
/// gaps: vec![Gap { display: 1, from_video_s: 3.0, to_video_s: 4.0 }],
/// };
///
/// let text = changes_text(&data);
/// assert!(text.starts_with("120 frames compared on display(s) 1 from 00:00.000 to 00:04.000: 1 stretch(es) of change\n"));
/// assert!(text.contains("up to 25.0% of the screen"));
/// assert!(text.ends_with("display 1 has no video from 00:03.000 to 00:04.000\n"));
/// ```
#[must_use]
pub fn changes_text(data: &ChangesData) -> String {
let mut out = String::new();
let _ = writeln!(
out,
"{} frames compared on display(s) {} from {} to {}: {} stretch(es) of change",
data.frames_examined,
data.displays
.iter()
.map(ToString::to_string)
.collect::<Vec<_>>()
.join(", "),
timecode::format(data.from_video_s),
timecode::format(data.to_video_s),
data.spans.len()
);
for span in &data.spans {
let events = if span.events.is_empty() {
String::new()
} else {
format!(
" events {}",
span.events
.iter()
.map(|seq| format!("#{seq}"))
.collect::<Vec<_>>()
.join(" ")
)
};
let _ = writeln!(
out,
"display {} {} - {} ({:.2} s, {} frames) up to {:.1}% of the screen box {:.0}%,{:.0}% {:.0}%x{:.0}%{events}",
span.display,
timecode::format(span.start_video_s),
timecode::format(span.end_video_s),
span.end_video_s - span.start_video_s,
span.frames,
span.peak_share * 100.0,
span.bbox[0] * 100.0,
span.bbox[1] * 100.0,
span.bbox[2] * 100.0,
span.bbox[3] * 100.0,
);
}
for gap in &data.gaps {
let _ = writeln!(
out,
"display {} has no video from {} to {}",
gap.display,
timecode::format(gap.from_video_s),
timecode::format(gap.to_video_s)
);
}
out
}
/// Renders the answer to `sheet`: where the image was written, its size and how many frames it
/// has, then a line for each moment that had no frame and why.
///
/// # Example
///
/// ```rust
/// use komorebi_recorder::analysis::sheet::{SheetData, Skipped};
/// use komorebi_recorder::report::sheet_text;
///
/// let data = SheetData {
/// path: "sheets/00-01.000-00-09.000.png".to_string(),
/// width: 1450,
/// height: 560,
/// frames: Vec::new(),
/// skipped: vec![Skipped {
/// display: 2,
/// video_s: 0.5,
/// reason: "the display had no video then".to_string(),
/// }],
/// };
///
/// assert_eq!(
/// sheet_text(&data),
/// "sheets/00-01.000-00-09.000.png (1450x560, 0 frames)\nno frame: display 2 at 00:00.500: the display had no video then\n",
/// );
/// ```
#[must_use]
pub fn sheet_text(data: &SheetData) -> String {
let mut out = String::new();
let _ = writeln!(
out,
"{} ({}x{}, {} frames)",
data.path,
data.width,
data.height,
data.frames.len()
);
for skipped in &data.skipped {
let _ = writeln!(
out,
"no frame: display {} at {}: {}",
skipped.display,
timecode::format(skipped.video_s),
skipped.reason
);
}
out
}
#[cfg(test)]
mod tests {
use super::*;
+286
View File
@@ -0,0 +1,286 @@
//! `changes` and `sheet` through the real binary, on sessions whose video says which frame it is.
use komorebi_recorder::testkit;
use komorebi_recorder::testkit::SessionBuilder;
use serde_json::Value;
use serde_json::json;
use std::process::Command;
use std::process::Output;
fn recorder(args: &[&str]) -> Output {
Command::new(env!("CARGO_BIN_EXE_komorebi-recorder"))
.args(args)
.output()
.expect("run komorebi-recorder")
}
fn json(args: &[&str]) -> Value {
let output = recorder(args);
assert!(
output.status.success(),
"{args:?} failed: {}",
String::from_utf8_lossy(&output.stderr)
);
serde_json::from_slice(&output.stdout).expect("JSON")
}
/// Display 1: 3 s at 30 fps that changes for a second, holds still for a second, then changes for
/// a second. Display 2: a 2 s video that only starts 1 s in. An event lands in the second change.
fn session(name: &str) -> Option<String> {
if !testkit::ffmpeg_available() {
eprintln!("skipped: ffmpeg is not available to make a test video");
return None;
}
let (session, _) = SessionBuilder::new(name)
.video_showing(1, 0.0, 90, 30, |frame| match frame {
0..=29 => frame,
30..=59 => 29,
_ => frame,
})
.video(2, 1.0, 60, 30)
.event(
2.05,
json!({"type": "FocusChange", "content": [{"type": "Accessibility", "content": "AXApplicationActivated"}, 5, 1]}),
None,
)
.build();
Some(session.dir.display().to_string())
}
#[test]
fn the_picture_changing_is_found_where_it_changed_and_not_where_it_held_still() {
let Some(dir) = session("video-changes") else {
return;
};
let answer = json(&["changes", &dir, "--display", "1", "--json"]);
assert_eq!(answer["schema"], "changes/1");
let spans = answer["data"]["spans"].as_array().expect("spans");
assert_eq!(spans.len(), 2, "{spans:#?}");
let first = &spans[0];
assert!(first["start_video_s"].as_f64().expect("start") < 0.1);
assert!((first["end_video_s"].as_f64().expect("end") - 0.97).abs() < 0.06);
let second = &spans[1];
assert!((second["start_video_s"].as_f64().expect("start") - 2.0).abs() < 0.06);
assert!((second["end_video_s"].as_f64().expect("end") - 2.97).abs() < 0.06);
// the event that landed in the second change is named there and not in the first
assert_eq!(second["events"], json!([1]));
assert_eq!(first["events"], json!([]));
// counting up changes the low bits, which are the blocks on the right of the picture, top to
// bottom
let bbox = second["bbox"].as_array().expect("bbox");
assert!(bbox[0].as_f64().expect("x") >= 0.5, "{bbox:?}");
assert!(bbox[2].as_f64().expect("width") > 0.3, "{bbox:?}");
assert!(bbox[3].as_f64().expect("height") > 0.9, "{bbox:?}");
assert!(second["bbox_points"].is_array());
assert!(second["peak_share"].as_f64().expect("share") > 0.0);
}
#[test]
fn a_display_with_no_video_for_part_of_the_range_has_a_gap() {
let Some(dir) = session("video-gaps") else {
return;
};
let answer = json(&["changes", &dir, "--from", "0", "--to", "4", "--json"]);
let gaps = answer["data"]["gaps"].as_array().expect("gaps");
let for_two: Vec<(f64, f64)> = gaps
.iter()
.filter(|gap| gap["display"] == 2)
.map(|gap| {
(
gap["from_video_s"].as_f64().expect("from"),
gap["to_video_s"].as_f64().expect("to"),
)
})
.collect();
assert_eq!(for_two.len(), 2, "{gaps:#?}");
assert!((for_two[0].0 - 0.0).abs() < 1e-6 && (for_two[0].1 - 1.0).abs() < 1e-6);
assert!((for_two[1].0 - 3.0).abs() < 1e-6 && (for_two[1].1 - 4.0).abs() < 1e-6);
// display 1 covers the first three seconds, so it only has a gap after them
assert!(gaps.iter().filter(|gap| gap["display"] == 1).count() <= 1);
assert_eq!(
recorder(&["changes", &dir, "--display", "9"]).status.code(),
Some(3)
);
assert_eq!(
recorder(&["changes", &dir, "--sensitivity", "0"])
.status
.code(),
Some(2)
);
}
/// Read what the sheet shows in a cell back out of the PNG as a frame number.
fn frame_shown_in(png_path: &str, frame: &Value) -> u32 {
let decoder = png::Decoder::new(std::io::BufReader::new(
std::fs::File::open(png_path).expect("png"),
));
let mut reader = decoder.read_info().expect("info");
let mut pixels = vec![0; reader.output_buffer_size().expect("size")];
let info = reader.next_frame(&mut pixels).expect("frame");
let (x, y) = (
frame["x"].as_u64().expect("x") as usize,
frame["y"].as_u64().expect("y") as usize,
);
let (width, height) = (
frame["width"].as_u64().expect("width") as usize,
frame["height"].as_u64().expect("height") as usize,
);
let mut bgra = Vec::with_capacity(width * height * 4);
for row in 0..height {
for column in 0..width {
let at = ((y + row) * info.width as usize + x + column) * 4;
bgra.extend_from_slice(&[pixels[at + 2], pixels[at + 1], pixels[at], 255]);
}
}
testkit::numbered_frame_index(&bgra, width as u32, height as u32)
}
#[test]
fn a_sheet_shows_the_frame_that_was_on_screen_at_each_moment() {
let Some(dir) = session("video-sheet") else {
return;
};
let answer = json(&[
"sheet",
&dir,
"--display",
"1",
"--at",
"0.5",
"--at",
"1.5",
"--at",
"2.5",
"--width",
"128",
"--json",
]);
assert_eq!(answer["schema"], "sheet/1");
let path = answer["data"]["path"].as_str().expect("path");
assert!(path.ends_with(".png") && std::path::Path::new(path).is_file());
let frames = answer["data"]["frames"].as_array().expect("frames");
assert_eq!(frames.len(), 3);
// 0.5 s is frame 15; the picture then holds on frame 29's until 2.0 s; 2.5 s is frame 75
assert_eq!(frame_shown_in(path, &frames[0]), 15);
assert_eq!(frame_shown_in(path, &frames[1]), 29);
assert_eq!(frame_shown_in(path, &frames[2]), 75);
// each frame says when it really was on screen, never later than asked
for frame in frames {
let shown = frame["shown_video_s"].as_f64().expect("shown");
let asked = frame["requested_video_s"].as_f64().expect("asked");
assert!(shown <= asked + 1e-6 && asked - shown < 1.1, "{frame}");
}
}
#[test]
fn a_sheet_marks_the_bug_region_and_names_the_events_around_a_frame() {
let Some(dir) = session("video-sheet-bug") else {
return;
};
let bug = json(&[
"annotate",
&dir,
"add",
"--kind",
"bug",
"--start",
"2",
"--end",
"3",
"--text",
"the layout jumped",
"--json",
]);
let id = bug["data"]["annotation"]["id"]
.as_str()
.expect("id")
.to_string();
let answer = json(&[
"sheet",
&dir,
"--display",
"1",
"--bug",
&id,
"--count",
"4",
"--json",
]);
let frames = answer["data"]["frames"].as_array().expect("frames");
assert_eq!(frames.len(), 4);
// the range runs from before the region to its end; frames inside it are marked
let marked: Vec<bool> = frames
.iter()
.map(|frame| !frame["bugs"].as_array().expect("bugs").is_empty())
.collect();
assert_eq!(marked, [false, false, true, true]);
assert_eq!(frames[3]["bugs"], json!([id]));
// the focus change at 2.05 s is near the frame at 2.0 s
let near: Vec<&Value> = frames
.iter()
.filter(|frame| frame["events"] == json!([1]))
.collect();
assert_eq!(near.len(), 1, "{frames:#?}");
}
#[test]
fn a_moment_a_display_had_no_video_is_skipped_not_invented() {
let Some(dir) = session("video-sheet-gap") else {
return;
};
let answer = json(&[
"sheet",
&dir,
"--display",
"2",
"--at",
"0.5",
"--at",
"1.5",
"--json",
]);
assert_eq!(
answer["data"]["frames"].as_array().expect("frames").len(),
1
);
let skipped = answer["data"]["skipped"].as_array().expect("skipped");
assert_eq!(skipped.len(), 1);
assert_eq!(skipped[0]["display"], 2);
// nothing to show at all is not found
assert_eq!(
recorder(&["sheet", &dir, "--display", "2", "--at", "0.5"])
.status
.code(),
Some(3)
);
// a sheet that would be absurdly large is refused with how to fix it
let huge = recorder(&[
"sheet",
&dir,
"--count",
"400",
"--columns",
"40",
"--width",
"1000",
]);
assert_eq!(huge.status.code(), Some(2));
}