mirror of
https://github.com/LGUG2Z/komorebi.git
synced 2026-11-15 21:32:26 +01:00
feat(analysis): see what the screen did
When and where on each display the picture changed, found by decoding the video, and a contact sheet of frames with the times, events and bug region marked. A display with no video for part of a range has a gap there instead of an invented frame.
This commit is contained in:
1 parent
26963391a3
commit
069347060d
12 files changed
+2043
No files matched your search
Generated
+2
@@ -1907,6 +1907,7 @@ checksum = "e2db585e1d738fc771bf08a151420d3ed193d9d895a36df7f6f8a9456b911ddc"
|
||||
name = "komorebi-recorder"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"ab_glyph",
|
||||
"block2 0.6.2",
|
||||
"chrono",
|
||||
"clap",
|
||||
@@ -1917,6 +1918,7 @@ dependencies = [
|
||||
"dispatch2",
|
||||
"dunce",
|
||||
"eframe",
|
||||
"epaint_default_fonts",
|
||||
"libc",
|
||||
"objc2 0.6.4",
|
||||
"objc2-av-foundation",
|
||||
|
||||
@@ -45,6 +45,8 @@ and `event:12.end` when it finished.
|
||||
| `window ID\|TEXT` | One window's life: when it appeared, moved, was parked off screen, was really somewhere else, and which event did each |
|
||||
| `code event:N`, `code log:N` | The line of komorebi's source that wrote a log line, with the code around it |
|
||||
| `search TEXT` | Where a word appears: komorebi's log, events, incidents, annotations (and `--oslog` for the macOS trace) |
|
||||
| `changes` | When and where on each display the picture changed, and which events fall inside |
|
||||
| `sheet` | A contact sheet image of frames with times, events and the bug region marked |
|
||||
| `annotate`, `annotations` | Add, change and list bug regions, notes and findings |
|
||||
|
||||
### Who changed it
|
||||
|
||||
@@ -6,6 +6,7 @@ edition.workspace = true
|
||||
repository.workspace = true
|
||||
|
||||
[dependencies]
|
||||
ab_glyph = "0.2"
|
||||
chrono = { workspace = true }
|
||||
clap = { workspace = true }
|
||||
color-eyre = { workspace = true }
|
||||
@@ -14,6 +15,7 @@ ctrlc = { version = "3", features = ["termination"] }
|
||||
dirs = { workspace = true }
|
||||
dunce = { workspace = true }
|
||||
eframe = { workspace = true }
|
||||
epaint_default_fonts = "0.33"
|
||||
parking_lot = { workspace = true }
|
||||
png = "0.18"
|
||||
regex = "1"
|
||||
|
||||
@@ -0,0 +1,444 @@
|
||||
//! When, and where on the screen, the picture changed.
|
||||
//!
|
||||
//! komorebi's events say when it thought something happened; the video says when the screen
|
||||
//! actually did. Comparing consecutive frames finds the stretches where pixels changed, how much
|
||||
//! of the screen they covered and the box around them, so an investigator can see whether a layout
|
||||
//! change was instant or took half a second, whether a window jumped, and what else on screen was
|
||||
//! going on. Where an event falls inside a stretch of change it is named.
|
||||
//!
|
||||
//! [`Analysis::changes`] is what the `changes` command prints and what the viewer scans in the
|
||||
//! background ([`Analysis::changes_until`]) to draw the screen lane of its timeline.
|
||||
//! [`Analysis::video_gaps`] reports where a display had no video to compare at all.
|
||||
//!
|
||||
//! This is about pixels, not about komorebi's state: a [`ChangeSpan`] reports that the picture
|
||||
//! moved, not what moved. The differences between two komorebi states are in [`super::diff`].
|
||||
//!
|
||||
//! # How a change is found
|
||||
//!
|
||||
//! Each display is decoded separately and every frame is compared with the one before it:
|
||||
//!
|
||||
//! - Frames are scaled to 240 pixels wide and reduced to brightness (Rec. 601 weights).
|
||||
//! - A pixel has changed if its brightness moved by more than 24 (of 255).
|
||||
//! - A frame is a change if the share of its pixels that changed is at least 0.05% divided by the
|
||||
//! `sensitivity` asked for: with a sensitivity of 2, changes half as large are noticed, and with
|
||||
//! 0.5 only changes twice as large.
|
||||
//! - Changed frames less than 0.15 s apart belong to one [`ChangeSpan`].
|
||||
//! - A frame is compared with the one before it on the same display, across the start of a new
|
||||
//! segment too (a display that came back after being unplugged or asleep), so a screen that
|
||||
//! looks different after the break is a change at the boundary. A frame of another size than
|
||||
//! the one before it (a display that changed resolution) is not compared with it.
|
||||
//!
|
||||
//! Frames come when the screen changes, not on a schedule, so a display whose video starts less
|
||||
//! than 0.5 s after the range starts, or stops less than 0.5 s before it ends, is not reported as
|
||||
//! having a [`Gap`].
|
||||
//!
|
||||
//! Decoding needs a video decoder on the host: where there is none, the functions here that read
|
||||
//! frames return an error saying so, and everything else in a session can still be read.
|
||||
|
||||
use super::Analysis;
|
||||
use super::envelope;
|
||||
use super::video::DecodedFrame;
|
||||
use super::video::SessionVideo;
|
||||
use crate::av::Scale;
|
||||
use color_eyre::eyre;
|
||||
use schemars::JsonSchema;
|
||||
use serde::Serialize;
|
||||
|
||||
/// The schema name that identifies a [`ChangesData`] in an envelope (`changes/1`); listed by the
|
||||
/// `schema` command.
|
||||
pub const SCHEMA_CHANGES: &str = "changes/1";
|
||||
|
||||
/// Frames are compared at this width: enough to see a window move, cheap enough to scan minutes.
|
||||
const COMPARE_WIDTH: u32 = 240;
|
||||
|
||||
/// A pixel counts as changed if its brightness moved by more than this (out of 255).
|
||||
const PIXEL_THRESHOLD: i32 = 24;
|
||||
|
||||
/// A frame is a change if at least this share of its pixels changed, at sensitivity 1.
|
||||
const SHARE_THRESHOLD: f64 = 0.0005;
|
||||
|
||||
/// Frames of one change closer together than this are one change.
|
||||
const MERGE_GAP_S: f64 = 0.15;
|
||||
|
||||
/// A display whose first frame arrives this late, or whose video stops this early, is not
|
||||
/// reported as having a gap: frames come when the screen changes, not on a schedule.
|
||||
const MIN_GAP_S: f64 = 0.5;
|
||||
|
||||
/// An event this close to a change is said to coincide with it.
|
||||
const COINCIDE_S: f64 = 0.25;
|
||||
|
||||
/// A stretch of video in which the picture of one display changed.
|
||||
///
|
||||
/// A span of a single changed frame has `start_video_s == end_video_s`.
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, JsonSchema)]
|
||||
pub struct ChangeSpan {
|
||||
/// The id of the display, the same number as [`Monitor::id`](super::state::Monitor::id) and
|
||||
/// the `--display` of the recording.
|
||||
pub display: u32,
|
||||
/// When the first changed frame was on screen, in seconds of video time.
|
||||
pub start_video_s: f64,
|
||||
/// When the last changed frame was on screen, in seconds of video time.
|
||||
pub end_video_s: f64,
|
||||
/// Changed frames in the span: not the frames that fall between its start and end.
|
||||
pub frames: usize,
|
||||
/// The largest share of the screen that changed between two consecutive frames.
|
||||
pub peak_share: f64,
|
||||
/// The box around everything that changed, as fractions of the screen: `[x, y, width, height]`.
|
||||
/// The fractions are of the size of the last frame decoded for the display.
|
||||
pub bbox: [f64; 4],
|
||||
/// The same box in global display points, if the display's frame is known:
|
||||
/// `[x, y, width, height]`.
|
||||
pub bbox_points: Option<[f64; 4]>,
|
||||
/// The `seq` of each komorebi event received from 0.25 s before the span starts to 0.25 s
|
||||
/// after it ends. An event is received when its handler has finished.
|
||||
pub events: Vec<u64>,
|
||||
}
|
||||
|
||||
/// A stretch of a range in which a display had no video at all.
|
||||
///
|
||||
/// A display has none while it is unplugged or asleep, or after its capture failed.
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, JsonSchema)]
|
||||
pub struct Gap {
|
||||
/// The id of the display.
|
||||
pub display: u32,
|
||||
/// Where the gap starts, in seconds of video time.
|
||||
pub from_video_s: f64,
|
||||
/// Where the gap ends, in seconds of video time.
|
||||
pub to_video_s: f64,
|
||||
}
|
||||
|
||||
/// Where the picture changed on each display over a range of the recording: the `data` of the
|
||||
/// `changes` command.
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, JsonSchema)]
|
||||
pub struct ChangesData {
|
||||
/// The start of the range that was scanned, in seconds of video time.
|
||||
pub from_video_s: f64,
|
||||
/// The end of the range that was scanned, in seconds of video time.
|
||||
pub to_video_s: f64,
|
||||
/// The displays that were scanned, in the order asked for.
|
||||
pub displays: Vec<u32>,
|
||||
/// How many frames were decoded and looked at, over all displays. A scan that was cancelled
|
||||
/// counts only what it got through.
|
||||
pub frames_examined: usize,
|
||||
/// The changes, ordered by start time and then by display.
|
||||
pub spans: Vec<ChangeSpan>,
|
||||
/// Stretches of the range where a display had no video at all (unplugged, asleep). See
|
||||
/// [`Analysis::video_gaps`].
|
||||
pub gaps: Vec<Gap>,
|
||||
}
|
||||
|
||||
/// The brightness of every pixel of a BGRA frame.
|
||||
fn luma(frame: &DecodedFrame) -> Vec<u8> {
|
||||
frame
|
||||
.frame
|
||||
.bgra
|
||||
.as_chunks::<4>()
|
||||
.0
|
||||
.iter()
|
||||
.map(|pixel| {
|
||||
// Rec. 601 weights in integers
|
||||
((u32::from(pixel[2]) * 299 + u32::from(pixel[1]) * 587 + u32::from(pixel[0]) * 114)
|
||||
/ 1000) as u8
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
struct Open {
|
||||
start: f64,
|
||||
end: f64,
|
||||
frames: usize,
|
||||
peak: f64,
|
||||
bounds: (u32, u32, u32, u32),
|
||||
}
|
||||
|
||||
impl Analysis {
|
||||
/// Finds where the picture changed on each of `displays` in `from_s..=to_s` (seconds of video
|
||||
/// time).
|
||||
///
|
||||
/// `sensitivity` scales how little has to change for a frame to count: the default of the
|
||||
/// `changes` command is `1.0`, a larger number notices smaller changes and a smaller one only
|
||||
/// larger changes. How a change is found is described in the [module](self) documentation.
|
||||
/// A display with no video in the range contributes no spans and is reported in
|
||||
/// [`ChangesData::gaps`]; so is one that is not in the session at all.
|
||||
///
|
||||
/// This decodes every frame of the range, so its cost is that of decoding the video, which was
|
||||
/// measured at 250 to 970 frames per second on macOS (HEVC, depending on frame size). `video`
|
||||
/// belongs to the thread that made it (a `SessionVideo` is not `Send`), so a scan on another
|
||||
/// thread builds a new `SessionVideo`.
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// Returns an error if:
|
||||
///
|
||||
/// - `sensitivity` is not above zero (the usage exit kind);
|
||||
/// - a display's video file cannot be resolved inside the session, opened or decoded (missing,
|
||||
/// truncated or unreadable), or the host has no video decoder.
|
||||
///
|
||||
/// # Example
|
||||
///
|
||||
/// This needs recorded video, so it is not run here. It is covered by
|
||||
/// `tests/video.rs::the_picture_changing_is_found_where_it_changed_and_not_where_it_held_still`
|
||||
/// and `tests/video.rs::a_display_with_no_video_for_part_of_the_range_has_a_gap`, which build
|
||||
/// sessions with numbered videos and run the `changes` command over them.
|
||||
///
|
||||
/// ```rust,no_run
|
||||
/// use komorebi_recorder::analysis::Analysis;
|
||||
/// use komorebi_recorder::analysis::guard::SessionPath;
|
||||
/// use komorebi_recorder::analysis::video::SessionVideo;
|
||||
///
|
||||
/// let analysis = Analysis::open(std::path::Path::new("session-dir"))?;
|
||||
/// let guard = SessionPath::new(&analysis.session.dir)?;
|
||||
/// let video = SessionVideo::new(&analysis.manifest, &guard);
|
||||
///
|
||||
/// let data = analysis.changes(&video, &video.displays(), 0.0, 60.0, 1.0)?;
|
||||
/// for span in &data.spans {
|
||||
/// println!(
|
||||
/// "display {}: {:.2} s to {:.2} s, events {:?}",
|
||||
/// span.display, span.start_video_s, span.end_video_s, span.events
|
||||
/// );
|
||||
/// }
|
||||
/// # Ok::<(), Box<dyn std::error::Error>>(())
|
||||
/// ```
|
||||
pub fn changes(
|
||||
&self,
|
||||
video: &SessionVideo,
|
||||
displays: &[u32],
|
||||
from_s: f64,
|
||||
to_s: f64,
|
||||
sensitivity: f64,
|
||||
) -> eyre::Result<ChangesData> {
|
||||
self.changes_until(video, displays, from_s, to_s, sensitivity, None)
|
||||
}
|
||||
|
||||
/// Finds where the picture changed, like [`Analysis::changes`], and stops early with what it
|
||||
/// has found so far once `cancel` is set: for a scan that runs in the background and whose
|
||||
/// result is no longer wanted.
|
||||
///
|
||||
/// `cancel` is checked once per decoded frame, so a scan stops within a frame of it being set.
|
||||
/// A cancelled scan is not an error: it returns the spans of what it got through, and
|
||||
/// [`ChangesData::frames_examined`] says how far that was. With `None` it runs to the end,
|
||||
/// which is what [`Analysis::changes`] does.
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// The same as [`Analysis::changes`].
|
||||
pub fn changes_until(
|
||||
&self,
|
||||
video: &SessionVideo,
|
||||
displays: &[u32],
|
||||
from_s: f64,
|
||||
to_s: f64,
|
||||
sensitivity: f64,
|
||||
cancel: Option<&std::sync::atomic::AtomicBool>,
|
||||
) -> eyre::Result<ChangesData> {
|
||||
if sensitivity <= 0.0 {
|
||||
return Err(envelope::usage("--sensitivity must be above zero"));
|
||||
}
|
||||
|
||||
let threshold = SHARE_THRESHOLD / sensitivity;
|
||||
let mut spans = Vec::new();
|
||||
let mut examined = 0;
|
||||
|
||||
for display in displays {
|
||||
let frame_box = self
|
||||
.manifest
|
||||
.displays
|
||||
.iter()
|
||||
.find(|entry| entry.result.display.id == *display)
|
||||
.map(|entry| entry.result.display.frame);
|
||||
|
||||
let mut previous: Option<(Vec<u8>, u32, u32)> = None;
|
||||
let mut open: Option<Open> = None;
|
||||
let mut finished: Vec<Open> = Vec::new();
|
||||
|
||||
examined += video.scan(
|
||||
*display,
|
||||
from_s,
|
||||
to_s,
|
||||
Scale::Width(COMPARE_WIDTH),
|
||||
&mut |decoded| {
|
||||
if cancel
|
||||
.is_some_and(|cancel| cancel.load(std::sync::atomic::Ordering::Relaxed))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
let now = luma(decoded);
|
||||
let (width, height) = (decoded.frame.width, decoded.frame.height);
|
||||
|
||||
if let Some((before, before_width, before_height)) = &previous
|
||||
&& (*before_width, *before_height) == (width, height)
|
||||
{
|
||||
let (mut count, mut min_x, mut min_y, mut max_x, mut max_y) =
|
||||
(0_usize, u32::MAX, u32::MAX, 0_u32, 0_u32);
|
||||
for (index, (a, b)) in before.iter().zip(&now).enumerate() {
|
||||
if (i32::from(*a) - i32::from(*b)).abs() > PIXEL_THRESHOLD {
|
||||
let (x, y) = (index as u32 % width, index as u32 / width);
|
||||
count += 1;
|
||||
min_x = min_x.min(x);
|
||||
max_x = max_x.max(x);
|
||||
min_y = min_y.min(y);
|
||||
max_y = max_y.max(y);
|
||||
}
|
||||
}
|
||||
|
||||
let share = count as f64 / now.len().max(1) as f64;
|
||||
if share >= threshold {
|
||||
match &mut open {
|
||||
Some(span) if decoded.shown_s - span.end <= MERGE_GAP_S => {
|
||||
span.end = decoded.shown_s;
|
||||
span.frames += 1;
|
||||
span.peak = span.peak.max(share);
|
||||
span.bounds = (
|
||||
span.bounds.0.min(min_x),
|
||||
span.bounds.1.min(min_y),
|
||||
span.bounds.2.max(max_x),
|
||||
span.bounds.3.max(max_y),
|
||||
);
|
||||
}
|
||||
_ => {
|
||||
finished.extend(open.take());
|
||||
open = Some(Open {
|
||||
start: decoded.shown_s,
|
||||
end: decoded.shown_s,
|
||||
frames: 1,
|
||||
peak: share,
|
||||
bounds: (min_x, min_y, max_x, max_y),
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
previous = Some((now, width, height));
|
||||
true
|
||||
},
|
||||
)?;
|
||||
finished.extend(open.take());
|
||||
|
||||
let dimensions = previous
|
||||
.as_ref()
|
||||
.map(|(_, width, height)| (*width, *height));
|
||||
for span in finished {
|
||||
let (width, height) = dimensions.unwrap_or((1, 1));
|
||||
let (w, h) = (f64::from(width), f64::from(height));
|
||||
let bbox = [
|
||||
f64::from(span.bounds.0) / w,
|
||||
f64::from(span.bounds.1) / h,
|
||||
f64::from(span.bounds.2 - span.bounds.0 + 1) / w,
|
||||
f64::from(span.bounds.3 - span.bounds.1 + 1) / h,
|
||||
];
|
||||
let events = self
|
||||
.events()
|
||||
.iter()
|
||||
.filter(|event| {
|
||||
let at = self.video_s(event.recv_mach_ns);
|
||||
at >= span.start - COINCIDE_S && at <= span.end + COINCIDE_S
|
||||
})
|
||||
.map(|event| event.seq)
|
||||
.collect();
|
||||
|
||||
spans.push(ChangeSpan {
|
||||
display: *display,
|
||||
start_video_s: span.start,
|
||||
end_video_s: span.end,
|
||||
frames: span.frames,
|
||||
peak_share: span.peak,
|
||||
bbox,
|
||||
bbox_points: frame_box.map(|[x, y, frame_width, frame_height]| {
|
||||
[
|
||||
x + bbox[0] * frame_width,
|
||||
y + bbox[1] * frame_height,
|
||||
bbox[2] * frame_width,
|
||||
bbox[3] * frame_height,
|
||||
]
|
||||
}),
|
||||
events,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
spans.sort_by(|a, b| {
|
||||
a.start_video_s
|
||||
.total_cmp(&b.start_video_s)
|
||||
.then(a.display.cmp(&b.display))
|
||||
});
|
||||
|
||||
Ok(ChangesData {
|
||||
from_video_s: from_s,
|
||||
to_video_s: to_s,
|
||||
displays: displays.to_vec(),
|
||||
frames_examined: examined,
|
||||
spans,
|
||||
gaps: self.video_gaps(displays, from_s, to_s),
|
||||
})
|
||||
}
|
||||
|
||||
/// Returns the stretches of `from_s..=to_s` (seconds of video time) in which each of
|
||||
/// `displays` has no video, ordered by display and then by time.
|
||||
///
|
||||
/// Only the manifest is read, no video is decoded. A segment of a display that recorded no
|
||||
/// frames does not count as video. A stretch shorter than 0.5 s is not a gap, because a display
|
||||
/// whose screen does not change produces no frames. A display that is not in the session has
|
||||
/// no video anywhere, so it gets one gap that is the whole range.
|
||||
///
|
||||
/// # Example
|
||||
///
|
||||
/// ```rust
|
||||
/// use komorebi_recorder::analysis::Analysis;
|
||||
/// use komorebi_recorder::analysis::changes::Gap;
|
||||
/// use komorebi_recorder::testkit::SessionBuilder;
|
||||
///
|
||||
/// // a ten second recording that has no video for display 2
|
||||
/// let (session, _) = SessionBuilder::new("doc-video-gaps").length(10.0).build();
|
||||
/// let analysis = Analysis::open(&session.dir)?;
|
||||
///
|
||||
/// assert_eq!(
|
||||
/// analysis.video_gaps(&[2], 0.0, 10.0),
|
||||
/// [Gap { display: 2, from_video_s: 0.0, to_video_s: 10.0 }],
|
||||
/// );
|
||||
/// # Ok::<(), Box<dyn std::error::Error>>(())
|
||||
/// ```
|
||||
///
|
||||
/// A display that has video for part of the range, and a gap after it, is covered by
|
||||
/// `tests/video.rs::a_display_with_no_video_for_part_of_the_range_has_a_gap`.
|
||||
#[must_use]
|
||||
pub fn video_gaps(&self, displays: &[u32], from_s: f64, to_s: f64) -> Vec<Gap> {
|
||||
let mut gaps = Vec::new();
|
||||
for display in displays {
|
||||
let mut covered: Vec<(f64, f64)> = self
|
||||
.manifest
|
||||
.displays
|
||||
.iter()
|
||||
.filter(|entry| entry.result.display.id == *display && entry.result.frames > 0)
|
||||
.map(|entry| {
|
||||
(
|
||||
entry.offset_s,
|
||||
entry.end_s.unwrap_or_else(|| self.manifest.video_span().1),
|
||||
)
|
||||
})
|
||||
.collect();
|
||||
covered.sort_by(|a, b| a.0.total_cmp(&b.0));
|
||||
|
||||
let mut cursor = from_s;
|
||||
for (start, end) in covered {
|
||||
if start > cursor + MIN_GAP_S && cursor < to_s {
|
||||
gaps.push(Gap {
|
||||
display: *display,
|
||||
from_video_s: cursor,
|
||||
to_video_s: start.min(to_s),
|
||||
});
|
||||
}
|
||||
cursor = cursor.max(end);
|
||||
}
|
||||
if cursor + MIN_GAP_S < to_s {
|
||||
gaps.push(Gap {
|
||||
display: *display,
|
||||
from_video_s: cursor,
|
||||
to_video_s: to_s,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
gaps
|
||||
}
|
||||
}
|
||||
@@ -12,6 +12,7 @@ pub mod blocks;
|
||||
pub mod brief;
|
||||
pub mod budget;
|
||||
pub mod catalog;
|
||||
pub mod changes;
|
||||
pub mod classify;
|
||||
pub mod code;
|
||||
pub mod diff;
|
||||
@@ -28,9 +29,11 @@ pub mod lookup;
|
||||
pub mod overview;
|
||||
pub mod schemas;
|
||||
pub mod search;
|
||||
pub mod sheet;
|
||||
pub mod state;
|
||||
pub mod timespec;
|
||||
pub mod trace;
|
||||
pub mod video;
|
||||
pub mod views;
|
||||
|
||||
pub use facade::Analysis;
|
||||
@@ -16,11 +16,13 @@ use super::answers::DiffData;
|
||||
use super::answers::EventsData;
|
||||
use super::answers::StateData;
|
||||
use super::brief::BriefData;
|
||||
use super::changes::ChangesData;
|
||||
use super::code::CodeData;
|
||||
use super::envelope::Envelope;
|
||||
use super::history::HistoryData;
|
||||
use super::overview::OverviewData;
|
||||
use super::search::SearchData;
|
||||
use super::sheet::SheetData;
|
||||
use super::trace::TraceData;
|
||||
use schemars::schema_for;
|
||||
use serde_json::Value;
|
||||
@@ -92,6 +94,16 @@ pub const ALL: &[Entry] = &[
|
||||
about: "`search`: where a word appears in the recording",
|
||||
schema: of::<SearchData>,
|
||||
},
|
||||
Entry {
|
||||
name: "changes/1",
|
||||
about: "`changes`: when and where the picture changed on each display",
|
||||
schema: of::<ChangesData>,
|
||||
},
|
||||
Entry {
|
||||
name: "sheet/1",
|
||||
about: "`sheet`: a contact sheet image and where each frame is on it",
|
||||
schema: of::<SheetData>,
|
||||
},
|
||||
Entry {
|
||||
name: "annotation/1",
|
||||
about: "`annotate add|edit`: the annotation as it now reads",
|
||||
|
||||
@@ -0,0 +1,549 @@
|
||||
//! A contact sheet: frames of the recording laid out on one image, with the times, the events and
|
||||
//! the bug regions marked.
|
||||
//!
|
||||
//! It is for someone (or an agent that can look at an image) to see what the screen did without
|
||||
//! playing a video.
|
||||
//!
|
||||
//! Each display is its own block of frames in time order. Every frame says when it is; where
|
||||
//! komorebi events fall around it they are named; frames inside a bug region the user marked have
|
||||
//! a red border. Frames are decoded at the width they are drawn, so a sheet of a long recording
|
||||
//! costs little.
|
||||
//!
|
||||
//! [`Analysis::sheet`] draws one from [`SheetOptions`] and a `SessionVideo` (the decoder of the
|
||||
//! session's video, in `analysis::video`) and returns a [`SheetData`] saying where each frame is on
|
||||
//! the image, so that a reader of the answer can find a frame without looking at the picture. The
|
||||
//! image is a PNG written to [`SheetOptions::out`]. Decoding needs a host that can decode the video
|
||||
//! (macOS and Windows; see `crate::av::supported`).
|
||||
//!
|
||||
//! # Layout
|
||||
//!
|
||||
//! From the top: a title with the session's directory name and the first and last of the moments;
|
||||
//! under it, if a bug region overlaps those moments, a red line with the first such region's id and
|
||||
//! its description (cut to 150 characters); then, for each display with a frame to show, a header
|
||||
//! naming it and its frames in rows of [`SheetOptions::columns`]. Under each frame is its timecode
|
||||
//! (red, with `BUG`, if it is inside a region) and the first two events received near it, as `#<seq>
|
||||
//! <name>`, with `+<n>` for any more. An event is near a frame if it was received within half the
|
||||
//! average spacing of the moments (half a second for a single moment) of the frame's moment, but
|
||||
//! never less than a tenth of a second.
|
||||
//!
|
||||
//! The drawn width of a frame is [`SheetOptions::cell_width`], limited to between 64 and 1920
|
||||
//! pixels, and the number of columns is limited to the number of moments. The image may be at most
|
||||
//! 8192 pixels wide and 16384 tall; a sheet that would be larger is refused (see
|
||||
//! [`Analysis::sheet`]).
|
||||
//!
|
||||
//! # Example
|
||||
//!
|
||||
//! Needs `ffmpeg` to make a test video, and a host that can decode it, so it is only compiled here;
|
||||
//! the tests `a_sheet_shows_the_frame_that_was_on_screen_at_each_moment` and
|
||||
//! `a_sheet_marks_the_bug_region_and_names_the_events_around_a_frame` in `tests/video.rs` run the
|
||||
//! same call through the `sheet` command.
|
||||
//!
|
||||
//! ```rust,no_run
|
||||
//! use komorebi_recorder::analysis::Analysis;
|
||||
//! use komorebi_recorder::analysis::sheet::SheetOptions;
|
||||
//! use komorebi_recorder::analysis::video::SessionVideo;
|
||||
//! use komorebi_recorder::testkit::SessionBuilder;
|
||||
//!
|
||||
//! // a recording of one display, 300 frames at 30 frames per second
|
||||
//! let (session, _) = SessionBuilder::new("doc-sheet").video(1, 0.0, 300, 30).build();
|
||||
//! let analysis = Analysis::open(&session.dir)?;
|
||||
//! let video = SessionVideo::new(&analysis.manifest, analysis.guard());
|
||||
//!
|
||||
//! let sheet = analysis.sheet(&video, &SheetOptions {
|
||||
//! times: vec![1.0, 5.0, 9.0],
|
||||
//! displays: video.displays(),
|
||||
//! cell_width: 240,
|
||||
//! columns: 3,
|
||||
//! regions: Vec::new(),
|
||||
//! out: session.path("sheets/example.png"),
|
||||
//! })?;
|
||||
//!
|
||||
//! assert_eq!(sheet.frames.len(), 3);
|
||||
//! assert!(sheet.skipped.is_empty());
|
||||
//! # Ok::<(), Box<dyn std::error::Error>>(())
|
||||
//! ```
|
||||
|
||||
use super::Analysis;
|
||||
use super::envelope;
|
||||
use super::video::SessionVideo;
|
||||
use crate::av::Scale;
|
||||
use crate::timecode;
|
||||
use ab_glyph::Font;
|
||||
use ab_glyph::FontRef;
|
||||
use ab_glyph::PxScale;
|
||||
use ab_glyph::ScaleFont;
|
||||
use ab_glyph::point;
|
||||
use color_eyre::eyre;
|
||||
use color_eyre::eyre::WrapErr;
|
||||
use schemars::JsonSchema;
|
||||
use serde::Serialize;
|
||||
use std::io::BufWriter;
|
||||
use std::path::Path;
|
||||
use std::path::PathBuf;
|
||||
|
||||
/// The `schema` of the `sheet` command's output: the name that [`SheetData`] has in an envelope.
|
||||
pub const SCHEMA_SHEET: &str = "sheet/1";
|
||||
|
||||
const BACKGROUND: [u8; 3] = [24, 24, 28];
|
||||
const TEXT: [u8; 3] = [225, 225, 230];
|
||||
const DIM: [u8; 3] = [140, 140, 150];
|
||||
const EVENT: [u8; 3] = [240, 190, 70];
|
||||
const BUG: [u8; 3] = [230, 70, 60];
|
||||
const MARGIN: u32 = 10;
|
||||
const LABEL_HEIGHT: u32 = 44;
|
||||
const HEADER_HEIGHT: u32 = 26;
|
||||
const TITLE_HEIGHT: u32 = 44;
|
||||
const MAX_WIDTH: u32 = 8192;
|
||||
const MAX_HEIGHT: u32 = 16384;
|
||||
|
||||
/// A bug region to mark on the sheet.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct Region {
|
||||
/// The annotation's id, written on the sheet and in [`SheetFrame::bugs`].
|
||||
pub id: String,
|
||||
/// Where the region starts, in seconds of video time.
|
||||
pub start_s: f64,
|
||||
/// Where the region ends, in seconds of video time. The `sheet` command gives an open-ended
|
||||
/// bug the end of the recording.
|
||||
pub end_s: f64,
|
||||
/// What the user said is wrong, written under the title of the sheet.
|
||||
pub text: String,
|
||||
}
|
||||
|
||||
/// What to put on a sheet, and where to write it.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct SheetOptions {
|
||||
/// The moments to show, in session video time. They are drawn in this order, so a caller that
|
||||
/// wants them in time order sorts them. Must not be empty.
|
||||
pub times: Vec<f64>,
|
||||
/// The displays to show, each as a block of its own. Must not be empty; a display listed that
|
||||
/// has no video at any of the moments is left out.
|
||||
pub displays: Vec<u32>,
|
||||
/// How wide each frame is drawn, in pixels; limited to between 64 and 1920.
|
||||
pub cell_width: u32,
|
||||
/// How many frames to a row; limited to between 1 and the number of moments.
|
||||
pub columns: usize,
|
||||
/// The bug regions to mark: a frame whose moment is inside one has a red border.
|
||||
pub regions: Vec<Region>,
|
||||
/// Where to write the PNG. Its parent directories are created, and a file already there is
|
||||
/// replaced.
|
||||
pub out: PathBuf,
|
||||
}
|
||||
|
||||
/// One frame of a sheet and where it is on the image.
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, JsonSchema)]
|
||||
pub struct SheetFrame {
|
||||
/// The display the frame is of.
|
||||
pub display: u32,
|
||||
/// The moment that was asked for, in seconds of video time.
|
||||
pub requested_video_s: f64,
|
||||
/// When the frame shown was actually put on screen, in seconds of video time. It is the picture
|
||||
/// that was on screen at the requested moment, so it is never later than it, and can be
|
||||
/// earlier: a still screen produces no new frames.
|
||||
pub shown_video_s: f64,
|
||||
/// Where it is on the image: pixels from the left edge to its left edge.
|
||||
pub x: u32,
|
||||
/// Where it is on the image: pixels from the top edge to its top edge.
|
||||
pub y: u32,
|
||||
/// How wide it is drawn, in pixels.
|
||||
pub width: u32,
|
||||
/// How tall it is drawn, in pixels.
|
||||
pub height: u32,
|
||||
/// The sequence numbers of the events received near it.
|
||||
pub events: Vec<u64>,
|
||||
/// The ids of the bug regions it is inside.
|
||||
pub bugs: Vec<String>,
|
||||
}
|
||||
|
||||
/// A moment of a display that the sheet has no frame for.
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, JsonSchema)]
|
||||
pub struct Skipped {
|
||||
/// The display that was asked for.
|
||||
pub display: u32,
|
||||
/// The moment that was asked for, in seconds of video time.
|
||||
pub video_s: f64,
|
||||
/// Why there is no frame.
|
||||
pub reason: String,
|
||||
}
|
||||
|
||||
/// The answer to `sheet`: the image that was written and where each frame is on it.
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, JsonSchema)]
|
||||
pub struct SheetData {
|
||||
/// Where the PNG was written.
|
||||
pub path: String,
|
||||
/// The width of the image, in pixels.
|
||||
pub width: u32,
|
||||
/// The height of the image, in pixels.
|
||||
pub height: u32,
|
||||
/// The frames on the image, a display at a time and in the order of the moments within it.
|
||||
pub frames: Vec<SheetFrame>,
|
||||
/// Moments a display had no video.
|
||||
pub skipped: Vec<Skipped>,
|
||||
}
|
||||
|
||||
struct Canvas {
|
||||
width: u32,
|
||||
height: u32,
|
||||
rgba: Vec<u8>,
|
||||
}
|
||||
|
||||
impl Canvas {
|
||||
fn new(width: u32, height: u32) -> Self {
|
||||
let mut rgba = Vec::with_capacity(width as usize * height as usize * 4);
|
||||
for _ in 0..width as usize * height as usize {
|
||||
rgba.extend_from_slice(&[BACKGROUND[0], BACKGROUND[1], BACKGROUND[2], 255]);
|
||||
}
|
||||
Self {
|
||||
width,
|
||||
height,
|
||||
rgba,
|
||||
}
|
||||
}
|
||||
|
||||
fn put(&mut self, x: i64, y: i64, color: [u8; 3], alpha: f32) {
|
||||
if x < 0 || y < 0 || x >= i64::from(self.width) || y >= i64::from(self.height) {
|
||||
return;
|
||||
}
|
||||
|
||||
let at = (y as usize * self.width as usize + x as usize) * 4;
|
||||
for (channel, wanted) in color.iter().enumerate() {
|
||||
let below = f32::from(self.rgba[at + channel]);
|
||||
self.rgba[at + channel] = (below + (f32::from(*wanted) - below) * alpha).round() as u8;
|
||||
}
|
||||
}
|
||||
|
||||
fn rectangle(&mut self, x: u32, y: u32, width: u32, height: u32, color: [u8; 3]) {
|
||||
for row in y..y + height {
|
||||
for column in x..x + width {
|
||||
self.put(i64::from(column), i64::from(row), color, 1.0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn border(&mut self, x: u32, y: u32, width: u32, height: u32, thickness: u32, color: [u8; 3]) {
|
||||
self.rectangle(x, y, width, thickness, color);
|
||||
self.rectangle(x, y + height - thickness, width, thickness, color);
|
||||
self.rectangle(x, y, thickness, height, color);
|
||||
self.rectangle(x + width - thickness, y, thickness, height, color);
|
||||
}
|
||||
|
||||
/// Copy a BGRA frame in at `x`, `y`.
|
||||
fn blit(&mut self, x: u32, y: u32, width: u32, height: u32, bgra: &[u8]) {
|
||||
for row in 0..height {
|
||||
for column in 0..width {
|
||||
let from = (row as usize * width as usize + column as usize) * 4;
|
||||
if from + 2 >= bgra.len() {
|
||||
return;
|
||||
}
|
||||
self.put(
|
||||
i64::from(x + column),
|
||||
i64::from(y + row),
|
||||
[bgra[from + 2], bgra[from + 1], bgra[from]],
|
||||
1.0,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn text(&mut self, font: &FontRef, size: f32, x: u32, y: u32, text: &str, color: [u8; 3]) {
|
||||
let scaled = font.as_scaled(PxScale::from(size));
|
||||
let baseline = y as f32 + scaled.ascent();
|
||||
let mut caret = x as f32;
|
||||
for character in text.chars() {
|
||||
let id = scaled.glyph_id(character);
|
||||
let glyph = id.with_scale_and_position(PxScale::from(size), point(caret, baseline));
|
||||
caret += scaled.h_advance(id);
|
||||
if let Some(outlined) = font.outline_glyph(glyph) {
|
||||
let bounds = outlined.px_bounds();
|
||||
outlined.draw(|gx, gy, coverage| {
|
||||
self.put(
|
||||
i64::from(bounds.min.x as i32) + i64::from(gx),
|
||||
i64::from(bounds.min.y as i32) + i64::from(gy),
|
||||
color,
|
||||
coverage,
|
||||
);
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn save(&self, path: &Path) -> eyre::Result<()> {
|
||||
if let Some(parent) = path.parent() {
|
||||
std::fs::create_dir_all(parent)
|
||||
.wrap_err_with(|| format!("could not create {}", parent.display()))?;
|
||||
}
|
||||
|
||||
let file = std::fs::File::create(path)
|
||||
.wrap_err_with(|| format!("could not create {}", path.display()))?;
|
||||
let mut encoder = png::Encoder::new(BufWriter::new(file), self.width, self.height);
|
||||
encoder.set_color(png::ColorType::Rgba);
|
||||
encoder.set_depth(png::BitDepth::Eight);
|
||||
let mut writer = encoder.write_header()?;
|
||||
writer.write_image_data(&self.rgba)?;
|
||||
writer.finish()?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
fn truncate(text: &str, limit: usize) -> String {
|
||||
let single: String = text.split_whitespace().collect::<Vec<_>>().join(" ");
|
||||
if single.chars().count() <= limit {
|
||||
single
|
||||
} else {
|
||||
let mut cut: String = single.chars().take(limit.saturating_sub(1)).collect();
|
||||
cut.push('…');
|
||||
cut
|
||||
}
|
||||
}
|
||||
|
||||
impl Analysis {
|
||||
/// Draws a contact sheet of the moments in `options` and writes it as a PNG.
|
||||
///
|
||||
/// Each frame is the one that was on screen at its moment, decoded at the drawn width. A moment
|
||||
/// at which a display had no video (it was unplugged, asleep, or not yet recorded) is skipped
|
||||
/// and listed in [`SheetData::skipped`] rather than failing the sheet. See the module docs for
|
||||
/// the layout.
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// - a usage error if `options.times` is empty, or if the sheet would be wider than 8192 or
|
||||
/// taller than 16384 pixels (the message says how big it would be; show fewer frames or draw
|
||||
/// them narrower);
|
||||
/// - an unusable error if `options.displays` is empty, which is what the commands pass for a
|
||||
/// session that recorded no video;
|
||||
/// - a not-found error if none of the moments has video on any of the displays;
|
||||
/// - an error if a video file cannot be opened or decoded (including on a host that cannot
|
||||
/// decode video), if the output directory or file cannot be created, or if the PNG cannot be
|
||||
/// written.
|
||||
///
|
||||
/// # Example
|
||||
///
|
||||
/// See the [module docs](self).
|
||||
pub fn sheet(&self, video: &SessionVideo, options: &SheetOptions) -> eyre::Result<SheetData> {
|
||||
if options.times.is_empty() {
|
||||
return Err(envelope::usage("there are no moments to show"));
|
||||
}
|
||||
if options.displays.is_empty() {
|
||||
return Err(envelope::unusable("the session recorded no video"));
|
||||
}
|
||||
|
||||
let font = FontRef::try_from_slice(epaint_default_fonts::HACK_REGULAR)
|
||||
.map_err(|_| eyre::Report::msg("the built-in font could not be read"))?;
|
||||
let columns = options.columns.clamp(1, options.times.len());
|
||||
let cell = options.cell_width.clamp(64, 1920);
|
||||
|
||||
// decode every frame first, so the layout knows each one's height
|
||||
struct Cell {
|
||||
requested: f64,
|
||||
shown: f64,
|
||||
width: u32,
|
||||
height: u32,
|
||||
bgra: Vec<u8>,
|
||||
}
|
||||
let mut blocks: Vec<(u32, Vec<Cell>)> = Vec::new();
|
||||
let mut skipped = Vec::new();
|
||||
for display in &options.displays {
|
||||
let mut cells = Vec::new();
|
||||
for time in &options.times {
|
||||
match video.frame_at(*display, *time, Scale::Width(cell))? {
|
||||
Some(decoded) => cells.push(Cell {
|
||||
requested: *time,
|
||||
shown: decoded.shown_s,
|
||||
width: decoded.frame.width,
|
||||
height: decoded.frame.height,
|
||||
bgra: decoded.frame.bgra,
|
||||
}),
|
||||
None => skipped.push(Skipped {
|
||||
display: *display,
|
||||
video_s: *time,
|
||||
reason:
|
||||
"the display had no video then (unplugged, asleep, or not yet recorded)"
|
||||
.to_string(),
|
||||
}),
|
||||
}
|
||||
}
|
||||
if !cells.is_empty() {
|
||||
blocks.push((*display, cells));
|
||||
}
|
||||
}
|
||||
if blocks.is_empty() {
|
||||
return Err(envelope::not_found(
|
||||
"none of those moments has video on any display",
|
||||
));
|
||||
}
|
||||
|
||||
let width = MARGIN + columns as u32 * (cell + MARGIN);
|
||||
let mut height = TITLE_HEIGHT + MARGIN;
|
||||
for (_, cells) in &blocks {
|
||||
height += HEADER_HEIGHT;
|
||||
for row in cells.chunks(columns) {
|
||||
height +=
|
||||
row.iter().map(|cell| cell.height).max().unwrap_or(0) + LABEL_HEIGHT + MARGIN;
|
||||
}
|
||||
}
|
||||
if width > MAX_WIDTH || height > MAX_HEIGHT {
|
||||
return Err(envelope::usage(format!(
|
||||
"a sheet of {} frames at width {cell} would be {width}x{height}: show fewer frames \
|
||||
or draw them narrower",
|
||||
options.times.len()
|
||||
)));
|
||||
}
|
||||
|
||||
let mut canvas = Canvas::new(width, height);
|
||||
let first = options.times.iter().copied().fold(f64::INFINITY, f64::min);
|
||||
let last = options
|
||||
.times
|
||||
.iter()
|
||||
.copied()
|
||||
.fold(f64::NEG_INFINITY, f64::max);
|
||||
let session_name = self
|
||||
.session
|
||||
.dir
|
||||
.file_name()
|
||||
.map_or_else(String::new, |name| name.to_string_lossy().into_owned());
|
||||
canvas.text(
|
||||
&font,
|
||||
15.0,
|
||||
MARGIN,
|
||||
6,
|
||||
&format!(
|
||||
"{session_name} {} - {}",
|
||||
timecode::format(first),
|
||||
timecode::format(last)
|
||||
),
|
||||
TEXT,
|
||||
);
|
||||
if let Some(region) = options
|
||||
.regions
|
||||
.iter()
|
||||
.find(|region| region.end_s >= first && region.start_s <= last)
|
||||
{
|
||||
canvas.text(
|
||||
&font,
|
||||
12.0,
|
||||
MARGIN,
|
||||
26,
|
||||
&format!("BUG {}: {}", region.id, truncate(®ion.text, 150)),
|
||||
BUG,
|
||||
);
|
||||
}
|
||||
|
||||
// spread of the moments, to know how near an event has to be to belong to a frame
|
||||
let step = if options.times.len() > 1 {
|
||||
(last - first) / (options.times.len() - 1) as f64
|
||||
} else {
|
||||
1.0
|
||||
};
|
||||
let reach = (step / 2.0).max(0.1);
|
||||
|
||||
let mut frames = Vec::new();
|
||||
let mut y = TITLE_HEIGHT + MARGIN;
|
||||
for (display, cells) in &blocks {
|
||||
canvas.text(
|
||||
&font,
|
||||
14.0,
|
||||
MARGIN,
|
||||
y + 4,
|
||||
&format!("display {display}"),
|
||||
DIM,
|
||||
);
|
||||
y += HEADER_HEIGHT;
|
||||
|
||||
for row in cells.chunks(columns) {
|
||||
let row_height = row.iter().map(|cell| cell.height).max().unwrap_or(0);
|
||||
for (column, cell_frame) in row.iter().enumerate() {
|
||||
let x = MARGIN + column as u32 * (cell + MARGIN);
|
||||
canvas.blit(x, y, cell_frame.width, cell_frame.height, &cell_frame.bgra);
|
||||
|
||||
let bugs: Vec<String> = options
|
||||
.regions
|
||||
.iter()
|
||||
.filter(|region| {
|
||||
cell_frame.requested >= region.start_s - 1e-6
|
||||
&& cell_frame.requested <= region.end_s + 1e-6
|
||||
})
|
||||
.map(|region| region.id.clone())
|
||||
.collect();
|
||||
if !bugs.is_empty() {
|
||||
canvas.border(x, y, cell_frame.width, cell_frame.height, 3, BUG);
|
||||
}
|
||||
|
||||
let events: Vec<u64> = self
|
||||
.events()
|
||||
.iter()
|
||||
.filter(|event| {
|
||||
(self.video_s(event.recv_mach_ns) - cell_frame.requested).abs() <= reach
|
||||
})
|
||||
.map(|event| event.seq)
|
||||
.collect();
|
||||
|
||||
let label_y = y + row_height + 3;
|
||||
canvas.text(
|
||||
&font,
|
||||
13.0,
|
||||
x,
|
||||
label_y,
|
||||
&timecode::format(cell_frame.requested),
|
||||
if bugs.is_empty() { TEXT } else { BUG },
|
||||
);
|
||||
if !bugs.is_empty() {
|
||||
canvas.text(&font, 12.0, x + 92, label_y, "BUG", BUG);
|
||||
}
|
||||
let named: Vec<String> = events
|
||||
.iter()
|
||||
.take(2)
|
||||
.filter_map(|seq| {
|
||||
let index = self.event_index(*seq)?;
|
||||
let event = &self.events()[index];
|
||||
Some(format!("#{seq} {}", event.info.name))
|
||||
})
|
||||
.collect();
|
||||
if !named.is_empty() {
|
||||
let more = events.len().saturating_sub(2);
|
||||
canvas.text(
|
||||
&font,
|
||||
11.0,
|
||||
x,
|
||||
label_y + 16,
|
||||
&truncate(
|
||||
&format!(
|
||||
"{}{}",
|
||||
named.join(" "),
|
||||
if more > 0 {
|
||||
format!(" +{more}")
|
||||
} else {
|
||||
String::new()
|
||||
}
|
||||
),
|
||||
(cell / 7) as usize,
|
||||
),
|
||||
EVENT,
|
||||
);
|
||||
}
|
||||
|
||||
frames.push(SheetFrame {
|
||||
display: *display,
|
||||
requested_video_s: cell_frame.requested,
|
||||
shown_video_s: cell_frame.shown,
|
||||
x,
|
||||
y,
|
||||
width: cell_frame.width,
|
||||
height: cell_frame.height,
|
||||
events,
|
||||
bugs,
|
||||
});
|
||||
}
|
||||
y += row_height + LABEL_HEIGHT + MARGIN;
|
||||
}
|
||||
}
|
||||
|
||||
canvas.save(&options.out)?;
|
||||
Ok(SheetData {
|
||||
path: options.out.display().to_string(),
|
||||
width,
|
||||
height,
|
||||
frames,
|
||||
skipped,
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,354 @@
|
||||
//! The recorded video, addressed by session time and display.
|
||||
//!
|
||||
//! A display's video is one file per segment (a display that was unplugged or slept and came
|
||||
//! back starts a new one), and a file's own time starts at its first frame. This maps a moment of
|
||||
//! session video time on a display to the file and file time that has it, decodes frames, and says
|
||||
//! when there is no video to show (the display was absent).
|
||||
//!
|
||||
//! [`SessionVideo`] is what the `changes` and `sheet` commands and the viewer's playback ask for
|
||||
//! frames. It uses the decoder of `crate::av` (`AVFoundation` on macOS, Media Foundation on Windows),
|
||||
//! so a host with neither cannot open a file; see `crate::av::supported`.
|
||||
//!
|
||||
//! # Times
|
||||
//!
|
||||
//! Every time here is *session video time*: seconds from the first frame of any display
|
||||
//! ([`crate::session::Manifest::video_t0`]). A segment's file has its own time, which starts at its
|
||||
//! first frame, and the manifest says how far into session time that is ([`DisplayEntry::offset_s`]):
|
||||
//! session time `t` is at `t - offset_s` in the file. A moment is covered by a segment of a display
|
||||
//! if it falls between the segment's start and end, with 50 ms of slack on each side, since
|
||||
//! segment boundaries are sampled and not exact; where two segments of a display both cover a
|
||||
//! moment, the later one is used ([`crate::session::Manifest::segment_at`]). A moment covered by
|
||||
//! none is a gap (the display was unplugged, asleep, or not yet recorded) and the methods say so
|
||||
//! with `None`, never with an invented frame.
|
||||
//!
|
||||
//! | To get | Use | Crosses segments |
|
||||
//! |---|---|---|
|
||||
//! | the frame on screen at one moment | [`SessionVideo::frame_at`] | no |
|
||||
//! | the frames from one moment on, one at a time | [`SessionVideo::open_at`] | no |
|
||||
//! | every frame of a range | [`SessionVideo::scan`] | yes |
|
||||
//! | which displays have video, and whether one does at a moment | [`SessionVideo::displays`], [`SessionVideo::has_video_at`] | |
|
||||
//!
|
||||
//! # Example
|
||||
//!
|
||||
//! Needs `ffmpeg` to make a test video, and a host that can decode it, so it is only compiled here;
|
||||
//! the tests of `tests/video.rs` (`a_sheet_shows_the_frame_that_was_on_screen_at_each_moment`,
|
||||
//! `a_moment_a_display_had_no_video_is_skipped_not_invented`) run these calls through the `sheet`
|
||||
//! command.
|
||||
//!
|
||||
//! ```rust,no_run
|
||||
//! use komorebi_recorder::analysis::Analysis;
|
||||
//! use komorebi_recorder::analysis::video::SessionVideo;
|
||||
//! use komorebi_recorder::av::Scale;
|
||||
//! use komorebi_recorder::testkit::SessionBuilder;
|
||||
//!
|
||||
//! // display 1 has video for 3 seconds from the start; display 2 only from 1 s to 3 s
|
||||
//! let (session, _) = SessionBuilder::new("doc-video")
|
||||
//! .video(1, 0.0, 90, 30)
|
||||
//! .video(2, 1.0, 60, 30)
|
||||
//! .build();
|
||||
//! let analysis = Analysis::open(&session.dir)?;
|
||||
//! let video = SessionVideo::new(&analysis.manifest, analysis.guard());
|
||||
//!
|
||||
//! assert_eq!(video.displays(), [1, 2]);
|
||||
//! assert!(video.has_video_at(1, 0.5));
|
||||
//! assert!(!video.has_video_at(2, 0.5));
|
||||
//!
|
||||
//! // the frame on screen at 1.5 s, at most 128 pixels wide; never shown later than asked
|
||||
//! let decoded = video.frame_at(1, 1.5, Scale::Width(128))?.expect("display 1 had video");
|
||||
//! assert!(decoded.shown_s <= 1.5);
|
||||
//!
|
||||
//! // display 2 had none at 0.5 s
|
||||
//! assert!(video.frame_at(2, 0.5, Scale::Width(128))?.is_none());
|
||||
//! # Ok::<(), Box<dyn std::error::Error>>(())
|
||||
//! ```
|
||||
|
||||
use super::guard::SessionPath;
|
||||
use crate::av::Asset;
|
||||
use crate::av::Frame;
|
||||
use crate::av::Scale;
|
||||
use crate::session::DisplayEntry;
|
||||
use crate::session::Manifest;
|
||||
use color_eyre::eyre;
|
||||
use std::cell::RefCell;
|
||||
use std::collections::HashMap;
|
||||
use std::path::PathBuf;
|
||||
use std::rc::Rc;
|
||||
|
||||
/// A decoded frame and when it was actually on screen.
|
||||
pub struct DecodedFrame {
|
||||
/// The display the frame is of.
|
||||
pub display: u32,
|
||||
/// The segment of the display's video it came from.
|
||||
pub segment: u32,
|
||||
/// When the frame was shown, in session video time. Never later than what was asked for.
|
||||
///
|
||||
/// A screen that holds still produces no new frames, so the frame that was on screen at a
|
||||
/// moment can have been put there some time before it.
|
||||
pub shown_s: f64,
|
||||
/// The decoded picture, scaled as asked.
|
||||
pub frame: Frame,
|
||||
}
|
||||
|
||||
/// A started decode of one segment of one display. Frames come in order, each with the session
|
||||
/// video time it was on screen.
|
||||
///
|
||||
/// Made by [`SessionVideo::open_at`]. A stream stays within its segment: it does not continue into
|
||||
/// the next one.
|
||||
pub struct Stream {
|
||||
reader: crate::av::Reader,
|
||||
offset_s: f64,
|
||||
/// The segment of the display's video it reads.
|
||||
pub segment: u32,
|
||||
/// Session video time at which the segment ends.
|
||||
pub end_s: f64,
|
||||
}
|
||||
|
||||
impl Stream {
|
||||
/// Returns the next frame and when it was shown, in session video time, or `None` at the end of
|
||||
/// the segment (or of the range the stream was opened with).
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// Returns an error if the decoder fails.
|
||||
pub fn next_frame(&mut self) -> eyre::Result<Option<(f64, Frame)>> {
|
||||
Ok(self
|
||||
.reader
|
||||
.next_frame()?
|
||||
.map(|frame| (self.offset_s + frame.pts_s, frame)))
|
||||
}
|
||||
}
|
||||
|
||||
/// The video of a session. Not `Send`: decoding objects belong to the thread that made them.
|
||||
///
|
||||
/// A file is opened the first time a frame of it is asked for and kept open for the life of the
|
||||
/// `SessionVideo`, so asking for many frames of the same segment opens it once. It borrows the
|
||||
/// session's [`Manifest`] and [`SessionPath`] (`Analysis::manifest` and `Analysis::guard`), and the
|
||||
/// guard is what keeps a file named in the manifest from leading outside the session.
|
||||
///
|
||||
/// See the [module docs](self) for how session time maps to files and an example.
|
||||
///
|
||||
/// # Example: it stays on the thread that made it
|
||||
///
|
||||
/// The compiler refuses to send one to another thread:
|
||||
///
|
||||
/// ```rust,compile_fail,E0277
|
||||
/// use komorebi_recorder::analysis::video::SessionVideo;
|
||||
///
|
||||
/// fn send_to_another_thread<T: Send>() {}
|
||||
/// send_to_another_thread::<SessionVideo<'static>>();
|
||||
/// ```
|
||||
///
|
||||
/// A thread that decodes makes its own, from a manifest and a guard that can be shared (an
|
||||
/// [`Analysis`](super::Analysis) is `Send` and `Sync`):
|
||||
///
|
||||
/// ```rust
|
||||
/// use komorebi_recorder::analysis::Analysis;
|
||||
/// use komorebi_recorder::analysis::video::SessionVideo;
|
||||
/// use komorebi_recorder::testkit::SessionBuilder;
|
||||
/// use std::sync::Arc;
|
||||
///
|
||||
/// let (session, _) = SessionBuilder::new("doc-video-threads").build();
|
||||
/// let analysis = Arc::new(Analysis::open(&session.dir)?);
|
||||
///
|
||||
/// let shared = Arc::clone(&analysis);
|
||||
/// let displays = std::thread::spawn(move || {
|
||||
/// SessionVideo::new(&shared.manifest, shared.guard()).displays()
|
||||
/// })
|
||||
/// .join()
|
||||
/// .expect("the thread finished");
|
||||
/// assert!(displays.is_empty(), "a session recorded without video has no displays with video");
|
||||
/// # Ok::<(), Box<dyn std::error::Error>>(())
|
||||
/// ```
|
||||
pub struct SessionVideo<'a> {
|
||||
manifest: &'a Manifest,
|
||||
guard: &'a SessionPath,
|
||||
assets: RefCell<HashMap<PathBuf, Rc<Asset>>>,
|
||||
}
|
||||
|
||||
impl<'a> SessionVideo<'a> {
|
||||
/// Returns the video of the session described by `manifest`, resolving its files through
|
||||
/// `guard`. Nothing is opened until a frame is asked for.
|
||||
#[must_use]
|
||||
pub fn new(manifest: &'a Manifest, guard: &'a SessionPath) -> Self {
|
||||
Self {
|
||||
manifest,
|
||||
guard,
|
||||
assets: RefCell::new(HashMap::new()),
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns the ids of the displays that have any video (a segment with at least one frame), in
|
||||
/// ascending order.
|
||||
#[must_use]
|
||||
pub fn displays(&self) -> Vec<u32> {
|
||||
let mut ids: Vec<u32> = self
|
||||
.manifest
|
||||
.displays
|
||||
.iter()
|
||||
.filter(|entry| entry.result.frames > 0)
|
||||
.map(|entry| entry.result.display.id)
|
||||
.collect();
|
||||
ids.sort_unstable();
|
||||
ids.dedup();
|
||||
ids
|
||||
}
|
||||
|
||||
fn asset(&self, entry: &DisplayEntry) -> eyre::Result<Rc<Asset>> {
|
||||
let path = self.guard.resolve(&entry.result.file)?;
|
||||
if let Some(found) = self.assets.borrow().get(&path) {
|
||||
return Ok(found.clone());
|
||||
}
|
||||
|
||||
let asset = Rc::new(Asset::open(&path)?);
|
||||
self.assets.borrow_mut().insert(path, asset.clone());
|
||||
Ok(asset)
|
||||
}
|
||||
|
||||
/// Returns the frame of `display` on screen at session video time `video_s`, decoded at
|
||||
/// `scale`, or `None` if the display had no video then.
|
||||
///
|
||||
/// The frame's [`DecodedFrame::shown_s`] is the time it was put on screen, which is never later
|
||||
/// than `video_s` and can be earlier. A time at or past the end of the file asks for its last
|
||||
/// frame, not for something beyond it. `None` is also returned if the decoder produces no frame
|
||||
/// for the moment. The display's video is looked up as [`SessionVideo`]'s module docs describe,
|
||||
/// so a moment in the slack at the edge of a segment is answered with that segment's nearest
|
||||
/// frame.
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// Returns an error if the segment's file leads outside the session (a usage error), cannot be
|
||||
/// opened (it is missing or damaged, or the host cannot decode video at all), or cannot be
|
||||
/// decoded.
|
||||
pub fn frame_at(
|
||||
&self,
|
||||
display: u32,
|
||||
video_s: f64,
|
||||
scale: Scale,
|
||||
) -> eyre::Result<Option<DecodedFrame>> {
|
||||
let Some(entry) = self.manifest.segment_at(display, video_s) else {
|
||||
return Ok(None);
|
||||
};
|
||||
|
||||
let asset = self.asset(entry)?;
|
||||
// the last moment of a file is the last frame, not past it
|
||||
let file_s = (video_s - entry.offset_s)
|
||||
.max(0.0)
|
||||
.min((asset.duration_s() - 0.002).max(0.0));
|
||||
let mut reader = asset.start(file_s, Some(file_s + 0.5), scale)?;
|
||||
Ok(reader.next_frame()?.map(|frame| DecodedFrame {
|
||||
display,
|
||||
segment: entry.result.segment,
|
||||
shown_s: entry.offset_s + frame.pts_s,
|
||||
frame,
|
||||
}))
|
||||
}
|
||||
|
||||
/// Returns `true` if and only if `display` has video at session video time `video_s`: a segment
|
||||
/// of it covers the moment (see the [module docs](self)). It does not open any file.
|
||||
#[must_use]
|
||||
pub fn has_video_at(&self, display: u32, video_s: f64) -> bool {
|
||||
self.manifest.segment_at(display, video_s).is_some()
|
||||
}
|
||||
|
||||
/// Starts decoding the segment of `display` that has video at `video_s`, beginning with the
|
||||
/// frame on screen then and running to `to_s` (or the end of the segment). `None` if the
|
||||
/// display had no video then.
|
||||
///
|
||||
/// The stream ends at the end of the segment even if `to_s` is later: use
|
||||
/// [`SessionVideo::scan`] to read across segments. A `to_s` before `video_s` is read as
|
||||
/// `video_s`.
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// Returns an error under the same conditions as [`SessionVideo::frame_at`].
|
||||
pub fn open_at(
|
||||
&self,
|
||||
display: u32,
|
||||
video_s: f64,
|
||||
to_s: Option<f64>,
|
||||
scale: Scale,
|
||||
) -> eyre::Result<Option<Stream>> {
|
||||
let Some(entry) = self.manifest.segment_at(display, video_s) else {
|
||||
return Ok(None);
|
||||
};
|
||||
|
||||
let asset = self.asset(entry)?;
|
||||
let start = (video_s - entry.offset_s)
|
||||
.max(0.0)
|
||||
.min((asset.duration_s() - 0.002).max(0.0));
|
||||
let stop = to_s.map(|to| (to - entry.offset_s).max(start));
|
||||
Ok(Some(Stream {
|
||||
reader: asset.start(start, stop, scale)?,
|
||||
offset_s: entry.offset_s,
|
||||
segment: entry.result.segment,
|
||||
end_s: entry.end_s.unwrap_or_else(|| self.manifest.video_span().1),
|
||||
}))
|
||||
}
|
||||
|
||||
/// Decodes every frame of `display` from `from_s` to `to_s`, across segments, in order, calling
|
||||
/// `visit` with each until it returns `false`. Returns how many frames were visited, counting
|
||||
/// the one `visit` stopped on.
|
||||
///
|
||||
/// A frame shown before `from_s` is visited only if it was shown less than half a second before
|
||||
/// it: the decoder starts at the frame that was on screen at `from_s`, which can be earlier than
|
||||
/// it, and that frame is not visited if it is older than that. No frame shown after `to_s` is
|
||||
/// visited. A gap between segments is not visited: there is nothing to decode there.
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// Returns an error under the same conditions as [`SessionVideo::frame_at`], for the first
|
||||
/// segment that fails. Frames visited before it stay visited.
|
||||
pub fn scan(
|
||||
&self,
|
||||
display: u32,
|
||||
from_s: f64,
|
||||
to_s: f64,
|
||||
scale: Scale,
|
||||
visit: &mut dyn FnMut(&DecodedFrame) -> bool,
|
||||
) -> eyre::Result<usize> {
|
||||
let mut segments: Vec<&DisplayEntry> = self
|
||||
.manifest
|
||||
.displays
|
||||
.iter()
|
||||
.filter(|entry| entry.result.display.id == display && entry.result.frames > 0)
|
||||
.collect();
|
||||
segments.sort_by(|a, b| a.offset_s.total_cmp(&b.offset_s));
|
||||
let (_, session_end) = self.manifest.video_span();
|
||||
|
||||
let mut visited = 0;
|
||||
for entry in segments {
|
||||
let end = entry.end_s.unwrap_or(session_end);
|
||||
if end < from_s || entry.offset_s > to_s {
|
||||
continue;
|
||||
}
|
||||
|
||||
let asset = self.asset(entry)?;
|
||||
let start = (from_s - entry.offset_s).max(0.0);
|
||||
let stop = (to_s - entry.offset_s).min(asset.duration_s());
|
||||
let mut reader = asset.start(start, Some(stop), scale)?;
|
||||
while let Some(frame) = reader.next_frame()? {
|
||||
let shown_s = entry.offset_s + frame.pts_s;
|
||||
// the decoder starts at the frame on screen at `from_s`, which can be before it
|
||||
if shown_s + 1e-6 < from_s && shown_s + 0.5 < from_s {
|
||||
continue;
|
||||
}
|
||||
if shown_s > to_s {
|
||||
break;
|
||||
}
|
||||
|
||||
visited += 1;
|
||||
let decoded = DecodedFrame {
|
||||
display,
|
||||
segment: entry.result.segment,
|
||||
shown_s,
|
||||
frame,
|
||||
};
|
||||
if !visit(&decoded) {
|
||||
return Ok(visited);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(visited)
|
||||
}
|
||||
}
|
||||
@@ -11,6 +11,7 @@ mod doctor;
|
||||
mod investigate;
|
||||
mod query;
|
||||
mod record;
|
||||
mod video;
|
||||
|
||||
use crate::analysis::envelope::unusable;
|
||||
use crate::analysis::envelope::with_hint;
|
||||
@@ -43,6 +44,8 @@ pub use query::LocateArgs;
|
||||
pub use query::RangeArgs;
|
||||
pub use query::SummaryArgs;
|
||||
pub use record::RecordArgs;
|
||||
pub use video::ChangesArgs;
|
||||
pub use video::SheetArgs;
|
||||
|
||||
/// The command line of `komorebi-recorder`.
|
||||
#[derive(Parser)]
|
||||
@@ -105,6 +108,10 @@ pub enum Command {
|
||||
Window(WindowArgs),
|
||||
/// One event end to end: what its handler logged, what changed, and who changed it
|
||||
Trace(TraceArgs),
|
||||
/// When and where on each display the picture changed
|
||||
Changes(ChangesArgs),
|
||||
/// A contact sheet image of frames, with the times, events and bug region marked
|
||||
Sheet(SheetArgs),
|
||||
/// Which line of komorebi's source wrote a line of its log, with the code around it
|
||||
Code(CodeArgs),
|
||||
/// Add, edit or delete a session's bug regions, notes and an agent's findings
|
||||
@@ -171,6 +178,8 @@ impl Command {
|
||||
Self::Code(args) => args.json,
|
||||
Self::Trace(args) => args.json,
|
||||
Self::Window(args) => args.json,
|
||||
Self::Changes(args) => args.json,
|
||||
Self::Sheet(args) => args.json,
|
||||
Self::Annotate(args) => args.json(),
|
||||
Self::Annotations(args) => args.json,
|
||||
Self::Doctor(args) => args.json,
|
||||
@@ -229,6 +238,8 @@ pub fn run(command: Command) -> eyre::Result<()> {
|
||||
Command::Code(args) => investigate::code(&args),
|
||||
Command::Trace(args) => investigate::trace(&args),
|
||||
Command::Window(args) => investigate::window(&args),
|
||||
Command::Changes(args) => video::changes(&args),
|
||||
Command::Sheet(args) => video::sheet(&args),
|
||||
Command::Annotate(args) => annotate::annotate(&args),
|
||||
Command::Annotations(args) => annotate::list(&args),
|
||||
Command::Reindex(args) => reindex(&args),
|
||||
|
||||
@@ -0,0 +1,235 @@
|
||||
//! `changes` and `sheet`: what the screen did.
|
||||
//!
|
||||
//! | Command | Asks | Answer, with `--json` |
|
||||
//! |---|---|---|
|
||||
//! | `changes` | [`Analysis::changes`](crate::analysis::Analysis::changes) | `changes/1` |
|
||||
//! | `sheet` | [`Analysis::sheet`] | `sheet/1` |
|
||||
//!
|
||||
//! Both work on a range of video time, given as `--from` and `--to` (any time `state --at` takes;
|
||||
//! the default is the whole video) or, instead of those, as `--bug <id>`: a bug region from three
|
||||
//! seconds before it starts to its end. Both work on the displays that have video (`--display`, repeatable, to pick),
|
||||
//! and both refuse a session that recorded no video. `sheet` also takes `--at` for exact moments, or
|
||||
//! `--count` moments evenly spaced across the range, and writes a PNG (by default to
|
||||
//! `<session>/sheets/<range>.png`); see [`crate::analysis::sheet`] for the layout.
|
||||
//!
|
||||
//! Needs a host that can decode the video (macOS and Windows; see [`crate::av::supported`]).
|
||||
//! Covered by `tests/video.rs`, which runs the commands through the real binary on a session with a
|
||||
//! test video (it skips with a message when `ffmpeg` is not there to make one).
|
||||
|
||||
use super::print_json;
|
||||
use crate::analysis::Analysis;
|
||||
use crate::analysis::changes::SCHEMA_CHANGES;
|
||||
use crate::analysis::envelope;
|
||||
use crate::analysis::envelope::Envelope;
|
||||
use crate::analysis::sheet::Region;
|
||||
use crate::analysis::sheet::SCHEMA_SHEET;
|
||||
use crate::analysis::sheet::SheetOptions;
|
||||
use crate::analysis::video::SessionVideo;
|
||||
use crate::annotations;
|
||||
use crate::report;
|
||||
use clap::Args;
|
||||
use color_eyre::eyre;
|
||||
use std::path::PathBuf;
|
||||
|
||||
/// Seconds shown before a bug region when a sheet is made of one.
|
||||
const BUG_PRE_ROLL_S: f64 = 3.0;
|
||||
|
||||
/// The arguments of `changes`.
|
||||
#[derive(Args, Debug)]
|
||||
pub struct ChangesArgs {
|
||||
/// Session directory
|
||||
pub session: PathBuf,
|
||||
/// From this moment [default: the start of the video]
|
||||
#[arg(long, value_name = "TIME")]
|
||||
pub from: Option<String>,
|
||||
/// To this moment [default: the end of the video]
|
||||
#[arg(long, value_name = "TIME")]
|
||||
pub to: Option<String>,
|
||||
/// Instead: a bug region, from a few seconds before it to its end
|
||||
#[arg(long, value_name = "ID", conflicts_with_all = ["from", "to"])]
|
||||
pub bug: Option<String>,
|
||||
/// Only this display (repeatable) [default: every display with video]
|
||||
#[arg(long = "display", value_name = "ID")]
|
||||
pub displays: Vec<u32>,
|
||||
/// Above 1 notices smaller changes, below 1 only larger ones
|
||||
#[arg(long, default_value_t = 1.0)]
|
||||
pub sensitivity: f64,
|
||||
/// Machine-readable JSON output
|
||||
#[arg(long)]
|
||||
pub json: bool,
|
||||
}
|
||||
|
||||
/// The arguments of `sheet`.
|
||||
#[derive(Args, Debug)]
|
||||
pub struct SheetArgs {
|
||||
/// Session directory
|
||||
pub session: PathBuf,
|
||||
/// From this moment [default: the start of the video]
|
||||
#[arg(long, value_name = "TIME")]
|
||||
pub from: Option<String>,
|
||||
/// To this moment [default: the end of the video]
|
||||
#[arg(long, value_name = "TIME")]
|
||||
pub to: Option<String>,
|
||||
/// Instead: a bug region, from a few seconds before it to its end
|
||||
#[arg(long, value_name = "ID", conflicts_with_all = ["from", "to"])]
|
||||
pub bug: Option<String>,
|
||||
/// Show exactly these moments instead of evenly spaced ones (repeatable)
|
||||
#[arg(long = "at", value_name = "TIME")]
|
||||
pub at: Vec<String>,
|
||||
/// How many frames, evenly spaced
|
||||
#[arg(long, default_value_t = 12)]
|
||||
pub count: usize,
|
||||
/// Only this display (repeatable) [default: every display with video]
|
||||
#[arg(long = "display", value_name = "ID")]
|
||||
pub displays: Vec<u32>,
|
||||
/// How wide each frame is drawn, in pixels
|
||||
#[arg(long, default_value_t = 480)]
|
||||
pub width: u32,
|
||||
/// Frames per row
|
||||
#[arg(long, default_value_t = 4)]
|
||||
pub columns: usize,
|
||||
/// Where to write the PNG [default: in the session's sheets/ directory, named by the range]
|
||||
#[arg(long, value_name = "FILE")]
|
||||
pub out: Option<PathBuf>,
|
||||
/// Machine-readable JSON output
|
||||
#[arg(long)]
|
||||
pub json: bool,
|
||||
}
|
||||
|
||||
/// The video-time range to work on: a bug region, or `--from`/`--to`, or everything.
|
||||
fn range(
|
||||
analysis: &Analysis,
|
||||
from: Option<&str>,
|
||||
to: Option<&str>,
|
||||
bug: Option<&str>,
|
||||
) -> eyre::Result<(f64, f64)> {
|
||||
if let Some(id) = bug {
|
||||
let loaded = annotations::load(&analysis.session.dir)?;
|
||||
let found = loaded
|
||||
.get(id)
|
||||
.ok_or_else(|| envelope::not_found(format!("there is no annotation {id}")))?;
|
||||
let end = found.end_or_session_end(&analysis.manifest);
|
||||
return Ok((
|
||||
(analysis.video_s(found.start_mach_ns) - BUG_PRE_ROLL_S).max(0.0),
|
||||
analysis.video_s(end),
|
||||
));
|
||||
}
|
||||
|
||||
let resolve = |text: Option<&str>, default: f64| -> eyre::Result<f64> {
|
||||
text.map_or(Ok(default), |text| {
|
||||
analysis.resolve(text).map(|resolved| resolved.video_s)
|
||||
})
|
||||
};
|
||||
let (_, end) = analysis.manifest.video_span();
|
||||
let (from, to) = (resolve(from, 0.0)?, resolve(to, end)?);
|
||||
if to < from {
|
||||
return Err(envelope::usage("the end of the range is before its start"));
|
||||
}
|
||||
|
||||
Ok((from, to))
|
||||
}
|
||||
|
||||
fn displays(video: &SessionVideo, wanted: &[u32]) -> eyre::Result<Vec<u32>> {
|
||||
let available = video.displays();
|
||||
if available.is_empty() {
|
||||
return Err(envelope::unusable("the session recorded no video"));
|
||||
}
|
||||
if wanted.is_empty() {
|
||||
return Ok(available);
|
||||
}
|
||||
|
||||
for display in wanted {
|
||||
if !available.contains(display) {
|
||||
return Err(envelope::not_found(format!(
|
||||
"display {display} has no video; displays with video: {available:?}"
|
||||
)));
|
||||
}
|
||||
}
|
||||
Ok(wanted.to_vec())
|
||||
}
|
||||
|
||||
pub(super) fn changes(args: &ChangesArgs) -> eyre::Result<()> {
|
||||
let analysis = Analysis::open(&args.session)?;
|
||||
let video = SessionVideo::new(&analysis.manifest, analysis.guard());
|
||||
let (from, to) = range(
|
||||
&analysis,
|
||||
args.from.as_deref(),
|
||||
args.to.as_deref(),
|
||||
args.bug.as_deref(),
|
||||
)?;
|
||||
let shown = displays(&video, &args.displays)?;
|
||||
|
||||
let data = analysis.changes(&video, &shown, from, to, args.sensitivity)?;
|
||||
if args.json {
|
||||
return print_json(&Envelope::ok(SCHEMA_CHANGES, data).warnings(analysis.warnings()));
|
||||
}
|
||||
|
||||
print!("{}", report::changes_text(&data));
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(super) fn sheet(args: &SheetArgs) -> eyre::Result<()> {
|
||||
let analysis = Analysis::open(&args.session)?;
|
||||
let video = SessionVideo::new(&analysis.manifest, analysis.guard());
|
||||
let (from, to) = range(
|
||||
&analysis,
|
||||
args.from.as_deref(),
|
||||
args.to.as_deref(),
|
||||
args.bug.as_deref(),
|
||||
)?;
|
||||
let shown = displays(&video, &args.displays)?;
|
||||
|
||||
let times: Vec<f64> = if args.at.is_empty() {
|
||||
let count = args.count.max(1);
|
||||
if count == 1 {
|
||||
vec![from]
|
||||
} else {
|
||||
(0..count)
|
||||
.map(|index| from + (to - from) * index as f64 / (count - 1) as f64)
|
||||
.collect()
|
||||
}
|
||||
} else {
|
||||
args.at
|
||||
.iter()
|
||||
.map(|text| analysis.resolve(text).map(|resolved| resolved.video_s))
|
||||
.collect::<eyre::Result<_>>()?
|
||||
};
|
||||
|
||||
let regions: Vec<Region> = annotations::load(&analysis.session.dir)?
|
||||
.bugs()
|
||||
.map(|bug| Region {
|
||||
id: bug.id.clone(),
|
||||
start_s: analysis.video_s(bug.start_mach_ns),
|
||||
end_s: analysis.video_s(bug.end_or_session_end(&analysis.manifest)),
|
||||
text: bug.text.clone(),
|
||||
})
|
||||
.collect();
|
||||
|
||||
let out = args.out.clone().unwrap_or_else(|| {
|
||||
analysis.session.path(format!(
|
||||
"sheets/{}-{}.png",
|
||||
crate::timecode::format(times.iter().copied().fold(f64::INFINITY, f64::min))
|
||||
.replace(':', "-"),
|
||||
crate::timecode::format(times.iter().copied().fold(f64::NEG_INFINITY, f64::max))
|
||||
.replace(':', "-"),
|
||||
))
|
||||
});
|
||||
|
||||
let data = analysis.sheet(
|
||||
&video,
|
||||
&SheetOptions {
|
||||
times,
|
||||
displays: shown,
|
||||
cell_width: args.width,
|
||||
columns: args.columns,
|
||||
regions,
|
||||
out,
|
||||
},
|
||||
)?;
|
||||
if args.json {
|
||||
return print_json(&Envelope::ok(SCHEMA_SHEET, data).warnings(analysis.warnings()));
|
||||
}
|
||||
|
||||
print!("{}", report::sheet_text(&data));
|
||||
Ok(())
|
||||
}
|
||||
@@ -31,6 +31,7 @@ use crate::analysis::answers::EventsData;
|
||||
use crate::analysis::answers::StateData;
|
||||
use crate::analysis::brief::BriefData;
|
||||
use crate::analysis::brief::RegionBrief;
|
||||
use crate::analysis::changes::ChangesData;
|
||||
use crate::analysis::code::CodeData;
|
||||
use crate::analysis::history::HistoryData;
|
||||
use crate::analysis::invariants::Severity;
|
||||
@@ -38,6 +39,7 @@ use crate::analysis::invariants::Violation;
|
||||
use crate::analysis::overview::Finding;
|
||||
use crate::analysis::overview::OverviewData;
|
||||
use crate::analysis::search::SearchData;
|
||||
use crate::analysis::sheet::SheetData;
|
||||
use crate::analysis::state::Container;
|
||||
use crate::analysis::state::Window;
|
||||
use crate::analysis::trace::TraceData;
|
||||
@@ -1554,6 +1556,147 @@ pub fn search_text(data: &SearchData) -> String {
|
||||
out
|
||||
}
|
||||
|
||||
/// Renders the answer to `changes`: how many frames were compared, then a line for each stretch of
|
||||
/// change, and the stretches in which a display had no video.
|
||||
///
|
||||
/// A stretch of change has its display, times and length, the most of the screen that changed
|
||||
/// between two frames, the box around what changed as percentages of the screen, and the events
|
||||
/// received during it.
|
||||
///
|
||||
/// # Example
|
||||
///
|
||||
/// ```rust
|
||||
/// use komorebi_recorder::analysis::changes::{ChangeSpan, ChangesData, Gap};
|
||||
/// use komorebi_recorder::report::changes_text;
|
||||
///
|
||||
/// let data = ChangesData {
|
||||
/// from_video_s: 0.0,
|
||||
/// to_video_s: 4.0,
|
||||
/// displays: vec![1],
|
||||
/// frames_examined: 120,
|
||||
/// spans: vec![ChangeSpan {
|
||||
/// display: 1,
|
||||
/// start_video_s: 1.0,
|
||||
/// end_video_s: 1.5,
|
||||
/// frames: 15,
|
||||
/// peak_share: 0.25,
|
||||
/// bbox: [0.5, 0.0, 0.5, 1.0],
|
||||
/// bbox_points: None,
|
||||
/// events: vec![3],
|
||||
/// }],
|
||||
/// gaps: vec![Gap { display: 1, from_video_s: 3.0, to_video_s: 4.0 }],
|
||||
/// };
|
||||
///
|
||||
/// let text = changes_text(&data);
|
||||
/// assert!(text.starts_with("120 frames compared on display(s) 1 from 00:00.000 to 00:04.000: 1 stretch(es) of change\n"));
|
||||
/// assert!(text.contains("up to 25.0% of the screen"));
|
||||
/// assert!(text.ends_with("display 1 has no video from 00:03.000 to 00:04.000\n"));
|
||||
/// ```
|
||||
#[must_use]
|
||||
pub fn changes_text(data: &ChangesData) -> String {
|
||||
let mut out = String::new();
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"{} frames compared on display(s) {} from {} to {}: {} stretch(es) of change",
|
||||
data.frames_examined,
|
||||
data.displays
|
||||
.iter()
|
||||
.map(ToString::to_string)
|
||||
.collect::<Vec<_>>()
|
||||
.join(", "),
|
||||
timecode::format(data.from_video_s),
|
||||
timecode::format(data.to_video_s),
|
||||
data.spans.len()
|
||||
);
|
||||
for span in &data.spans {
|
||||
let events = if span.events.is_empty() {
|
||||
String::new()
|
||||
} else {
|
||||
format!(
|
||||
" events {}",
|
||||
span.events
|
||||
.iter()
|
||||
.map(|seq| format!("#{seq}"))
|
||||
.collect::<Vec<_>>()
|
||||
.join(" ")
|
||||
)
|
||||
};
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"display {} {} - {} ({:.2} s, {} frames) up to {:.1}% of the screen box {:.0}%,{:.0}% {:.0}%x{:.0}%{events}",
|
||||
span.display,
|
||||
timecode::format(span.start_video_s),
|
||||
timecode::format(span.end_video_s),
|
||||
span.end_video_s - span.start_video_s,
|
||||
span.frames,
|
||||
span.peak_share * 100.0,
|
||||
span.bbox[0] * 100.0,
|
||||
span.bbox[1] * 100.0,
|
||||
span.bbox[2] * 100.0,
|
||||
span.bbox[3] * 100.0,
|
||||
);
|
||||
}
|
||||
for gap in &data.gaps {
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"display {} has no video from {} to {}",
|
||||
gap.display,
|
||||
timecode::format(gap.from_video_s),
|
||||
timecode::format(gap.to_video_s)
|
||||
);
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// Renders the answer to `sheet`: where the image was written, its size and how many frames it
|
||||
/// has, then a line for each moment that had no frame and why.
|
||||
///
|
||||
/// # Example
|
||||
///
|
||||
/// ```rust
|
||||
/// use komorebi_recorder::analysis::sheet::{SheetData, Skipped};
|
||||
/// use komorebi_recorder::report::sheet_text;
|
||||
///
|
||||
/// let data = SheetData {
|
||||
/// path: "sheets/00-01.000-00-09.000.png".to_string(),
|
||||
/// width: 1450,
|
||||
/// height: 560,
|
||||
/// frames: Vec::new(),
|
||||
/// skipped: vec![Skipped {
|
||||
/// display: 2,
|
||||
/// video_s: 0.5,
|
||||
/// reason: "the display had no video then".to_string(),
|
||||
/// }],
|
||||
/// };
|
||||
///
|
||||
/// assert_eq!(
|
||||
/// sheet_text(&data),
|
||||
/// "sheets/00-01.000-00-09.000.png (1450x560, 0 frames)\nno frame: display 2 at 00:00.500: the display had no video then\n",
|
||||
/// );
|
||||
/// ```
|
||||
#[must_use]
|
||||
pub fn sheet_text(data: &SheetData) -> String {
|
||||
let mut out = String::new();
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"{} ({}x{}, {} frames)",
|
||||
data.path,
|
||||
data.width,
|
||||
data.height,
|
||||
data.frames.len()
|
||||
);
|
||||
for skipped in &data.skipped {
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"no frame: display {} at {}: {}",
|
||||
skipped.display,
|
||||
timecode::format(skipped.video_s),
|
||||
skipped.reason
|
||||
);
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
@@ -0,0 +1,286 @@
|
||||
//! `changes` and `sheet` through the real binary, on sessions whose video says which frame it is.
|
||||
|
||||
use komorebi_recorder::testkit;
|
||||
use komorebi_recorder::testkit::SessionBuilder;
|
||||
use serde_json::Value;
|
||||
use serde_json::json;
|
||||
use std::process::Command;
|
||||
use std::process::Output;
|
||||
|
||||
fn recorder(args: &[&str]) -> Output {
|
||||
Command::new(env!("CARGO_BIN_EXE_komorebi-recorder"))
|
||||
.args(args)
|
||||
.output()
|
||||
.expect("run komorebi-recorder")
|
||||
}
|
||||
|
||||
fn json(args: &[&str]) -> Value {
|
||||
let output = recorder(args);
|
||||
assert!(
|
||||
output.status.success(),
|
||||
"{args:?} failed: {}",
|
||||
String::from_utf8_lossy(&output.stderr)
|
||||
);
|
||||
serde_json::from_slice(&output.stdout).expect("JSON")
|
||||
}
|
||||
|
||||
/// Display 1: 3 s at 30 fps that changes for a second, holds still for a second, then changes for
|
||||
/// a second. Display 2: a 2 s video that only starts 1 s in. An event lands in the second change.
|
||||
fn session(name: &str) -> Option<String> {
|
||||
if !testkit::ffmpeg_available() {
|
||||
eprintln!("skipped: ffmpeg is not available to make a test video");
|
||||
return None;
|
||||
}
|
||||
|
||||
let (session, _) = SessionBuilder::new(name)
|
||||
.video_showing(1, 0.0, 90, 30, |frame| match frame {
|
||||
0..=29 => frame,
|
||||
30..=59 => 29,
|
||||
_ => frame,
|
||||
})
|
||||
.video(2, 1.0, 60, 30)
|
||||
.event(
|
||||
2.05,
|
||||
json!({"type": "FocusChange", "content": [{"type": "Accessibility", "content": "AXApplicationActivated"}, 5, 1]}),
|
||||
None,
|
||||
)
|
||||
.build();
|
||||
Some(session.dir.display().to_string())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_picture_changing_is_found_where_it_changed_and_not_where_it_held_still() {
|
||||
let Some(dir) = session("video-changes") else {
|
||||
return;
|
||||
};
|
||||
|
||||
let answer = json(&["changes", &dir, "--display", "1", "--json"]);
|
||||
assert_eq!(answer["schema"], "changes/1");
|
||||
let spans = answer["data"]["spans"].as_array().expect("spans");
|
||||
assert_eq!(spans.len(), 2, "{spans:#?}");
|
||||
|
||||
let first = &spans[0];
|
||||
assert!(first["start_video_s"].as_f64().expect("start") < 0.1);
|
||||
assert!((first["end_video_s"].as_f64().expect("end") - 0.97).abs() < 0.06);
|
||||
let second = &spans[1];
|
||||
assert!((second["start_video_s"].as_f64().expect("start") - 2.0).abs() < 0.06);
|
||||
assert!((second["end_video_s"].as_f64().expect("end") - 2.97).abs() < 0.06);
|
||||
|
||||
// the event that landed in the second change is named there and not in the first
|
||||
assert_eq!(second["events"], json!([1]));
|
||||
assert_eq!(first["events"], json!([]));
|
||||
|
||||
// counting up changes the low bits, which are the blocks on the right of the picture, top to
|
||||
// bottom
|
||||
let bbox = second["bbox"].as_array().expect("bbox");
|
||||
assert!(bbox[0].as_f64().expect("x") >= 0.5, "{bbox:?}");
|
||||
assert!(bbox[2].as_f64().expect("width") > 0.3, "{bbox:?}");
|
||||
assert!(bbox[3].as_f64().expect("height") > 0.9, "{bbox:?}");
|
||||
assert!(second["bbox_points"].is_array());
|
||||
assert!(second["peak_share"].as_f64().expect("share") > 0.0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_display_with_no_video_for_part_of_the_range_has_a_gap() {
|
||||
let Some(dir) = session("video-gaps") else {
|
||||
return;
|
||||
};
|
||||
|
||||
let answer = json(&["changes", &dir, "--from", "0", "--to", "4", "--json"]);
|
||||
let gaps = answer["data"]["gaps"].as_array().expect("gaps");
|
||||
let for_two: Vec<(f64, f64)> = gaps
|
||||
.iter()
|
||||
.filter(|gap| gap["display"] == 2)
|
||||
.map(|gap| {
|
||||
(
|
||||
gap["from_video_s"].as_f64().expect("from"),
|
||||
gap["to_video_s"].as_f64().expect("to"),
|
||||
)
|
||||
})
|
||||
.collect();
|
||||
assert_eq!(for_two.len(), 2, "{gaps:#?}");
|
||||
assert!((for_two[0].0 - 0.0).abs() < 1e-6 && (for_two[0].1 - 1.0).abs() < 1e-6);
|
||||
assert!((for_two[1].0 - 3.0).abs() < 1e-6 && (for_two[1].1 - 4.0).abs() < 1e-6);
|
||||
|
||||
// display 1 covers the first three seconds, so it only has a gap after them
|
||||
assert!(gaps.iter().filter(|gap| gap["display"] == 1).count() <= 1);
|
||||
|
||||
assert_eq!(
|
||||
recorder(&["changes", &dir, "--display", "9"]).status.code(),
|
||||
Some(3)
|
||||
);
|
||||
assert_eq!(
|
||||
recorder(&["changes", &dir, "--sensitivity", "0"])
|
||||
.status
|
||||
.code(),
|
||||
Some(2)
|
||||
);
|
||||
}
|
||||
|
||||
/// Read what the sheet shows in a cell back out of the PNG as a frame number.
|
||||
fn frame_shown_in(png_path: &str, frame: &Value) -> u32 {
|
||||
let decoder = png::Decoder::new(std::io::BufReader::new(
|
||||
std::fs::File::open(png_path).expect("png"),
|
||||
));
|
||||
let mut reader = decoder.read_info().expect("info");
|
||||
let mut pixels = vec![0; reader.output_buffer_size().expect("size")];
|
||||
let info = reader.next_frame(&mut pixels).expect("frame");
|
||||
|
||||
let (x, y) = (
|
||||
frame["x"].as_u64().expect("x") as usize,
|
||||
frame["y"].as_u64().expect("y") as usize,
|
||||
);
|
||||
let (width, height) = (
|
||||
frame["width"].as_u64().expect("width") as usize,
|
||||
frame["height"].as_u64().expect("height") as usize,
|
||||
);
|
||||
let mut bgra = Vec::with_capacity(width * height * 4);
|
||||
for row in 0..height {
|
||||
for column in 0..width {
|
||||
let at = ((y + row) * info.width as usize + x + column) * 4;
|
||||
bgra.extend_from_slice(&[pixels[at + 2], pixels[at + 1], pixels[at], 255]);
|
||||
}
|
||||
}
|
||||
|
||||
testkit::numbered_frame_index(&bgra, width as u32, height as u32)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_sheet_shows_the_frame_that_was_on_screen_at_each_moment() {
|
||||
let Some(dir) = session("video-sheet") else {
|
||||
return;
|
||||
};
|
||||
|
||||
let answer = json(&[
|
||||
"sheet",
|
||||
&dir,
|
||||
"--display",
|
||||
"1",
|
||||
"--at",
|
||||
"0.5",
|
||||
"--at",
|
||||
"1.5",
|
||||
"--at",
|
||||
"2.5",
|
||||
"--width",
|
||||
"128",
|
||||
"--json",
|
||||
]);
|
||||
assert_eq!(answer["schema"], "sheet/1");
|
||||
let path = answer["data"]["path"].as_str().expect("path");
|
||||
assert!(path.ends_with(".png") && std::path::Path::new(path).is_file());
|
||||
|
||||
let frames = answer["data"]["frames"].as_array().expect("frames");
|
||||
assert_eq!(frames.len(), 3);
|
||||
// 0.5 s is frame 15; the picture then holds on frame 29's until 2.0 s; 2.5 s is frame 75
|
||||
assert_eq!(frame_shown_in(path, &frames[0]), 15);
|
||||
assert_eq!(frame_shown_in(path, &frames[1]), 29);
|
||||
assert_eq!(frame_shown_in(path, &frames[2]), 75);
|
||||
|
||||
// each frame says when it really was on screen, never later than asked
|
||||
for frame in frames {
|
||||
let shown = frame["shown_video_s"].as_f64().expect("shown");
|
||||
let asked = frame["requested_video_s"].as_f64().expect("asked");
|
||||
assert!(shown <= asked + 1e-6 && asked - shown < 1.1, "{frame}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_sheet_marks_the_bug_region_and_names_the_events_around_a_frame() {
|
||||
let Some(dir) = session("video-sheet-bug") else {
|
||||
return;
|
||||
};
|
||||
let bug = json(&[
|
||||
"annotate",
|
||||
&dir,
|
||||
"add",
|
||||
"--kind",
|
||||
"bug",
|
||||
"--start",
|
||||
"2",
|
||||
"--end",
|
||||
"3",
|
||||
"--text",
|
||||
"the layout jumped",
|
||||
"--json",
|
||||
]);
|
||||
let id = bug["data"]["annotation"]["id"]
|
||||
.as_str()
|
||||
.expect("id")
|
||||
.to_string();
|
||||
|
||||
let answer = json(&[
|
||||
"sheet",
|
||||
&dir,
|
||||
"--display",
|
||||
"1",
|
||||
"--bug",
|
||||
&id,
|
||||
"--count",
|
||||
"4",
|
||||
"--json",
|
||||
]);
|
||||
let frames = answer["data"]["frames"].as_array().expect("frames");
|
||||
assert_eq!(frames.len(), 4);
|
||||
|
||||
// the range runs from before the region to its end; frames inside it are marked
|
||||
let marked: Vec<bool> = frames
|
||||
.iter()
|
||||
.map(|frame| !frame["bugs"].as_array().expect("bugs").is_empty())
|
||||
.collect();
|
||||
assert_eq!(marked, [false, false, true, true]);
|
||||
assert_eq!(frames[3]["bugs"], json!([id]));
|
||||
|
||||
// the focus change at 2.05 s is near the frame at 2.0 s
|
||||
let near: Vec<&Value> = frames
|
||||
.iter()
|
||||
.filter(|frame| frame["events"] == json!([1]))
|
||||
.collect();
|
||||
assert_eq!(near.len(), 1, "{frames:#?}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_moment_a_display_had_no_video_is_skipped_not_invented() {
|
||||
let Some(dir) = session("video-sheet-gap") else {
|
||||
return;
|
||||
};
|
||||
|
||||
let answer = json(&[
|
||||
"sheet",
|
||||
&dir,
|
||||
"--display",
|
||||
"2",
|
||||
"--at",
|
||||
"0.5",
|
||||
"--at",
|
||||
"1.5",
|
||||
"--json",
|
||||
]);
|
||||
assert_eq!(
|
||||
answer["data"]["frames"].as_array().expect("frames").len(),
|
||||
1
|
||||
);
|
||||
let skipped = answer["data"]["skipped"].as_array().expect("skipped");
|
||||
assert_eq!(skipped.len(), 1);
|
||||
assert_eq!(skipped[0]["display"], 2);
|
||||
|
||||
// nothing to show at all is not found
|
||||
assert_eq!(
|
||||
recorder(&["sheet", &dir, "--display", "2", "--at", "0.5"])
|
||||
.status
|
||||
.code(),
|
||||
Some(3)
|
||||
);
|
||||
// a sheet that would be absurdly large is refused with how to fix it
|
||||
let huge = recorder(&[
|
||||
"sheet",
|
||||
&dir,
|
||||
"--count",
|
||||
"400",
|
||||
"--columns",
|
||||
"40",
|
||||
"--width",
|
||||
"1000",
|
||||
]);
|
||||
assert_eq!(huge.status.code(), Some(2));
|
||||
}
|
||||
Reference in new issue
Block a user