From 3c4f3528469b640d335ed2644b9ef76418a486ac Mon Sep 17 00:00:00 2001 From: LGUG2Z Date: Wed, 28 Oct 2026 15:48:52 -0700 Subject: [PATCH] feat(record): sample every window's rect at video rate, and draw them on sheets record --window-track[=HZ] (off by default, 30 Hz without a value, 1..=120) writes windows.jsonl: a sampler thread lists every top-level window once per tick and writes the full window list only when something changed, plus a keyframe each second, so a still desktop costs about a row a second. platform::window::list gives generic facts (Windows: EnumWindows + DWM cloaked/frame bounds; macOS: CGWindowList bounds and on-screen flag). The manifest records the track (rows, late ticks) and warns if it stopped or fell behind. Analysis::window_track answers 'windows at mach_ns T', and sheet --windows draws each window's rect and label on every frame (cloaked windows dim), or notes that the session has no track. The macOS part (platform/macos/apps.rs window_list) was written against the objc2 sources and is NOT built or run: this was developed on Windows. --- README.md | 2 + docs/PLATFORM.md | 16 + docs/usage/investigating-a-recording.md | 2 +- docs/usage/recording-a-bug-repro.md | 15 + komorebi-recorder/src/analysis/mod.rs | 1 + komorebi-recorder/src/analysis/sheet.rs | 174 +++++++ .../src/analysis/window_track.rs | 129 +++++ komorebi-recorder/src/cli/record.rs | 15 + komorebi-recorder/src/cli/video.rs | 25 +- komorebi-recorder/src/guide.md | 5 + komorebi-recorder/src/platform/macos/apps.rs | 99 ++++ komorebi-recorder/src/platform/unsupported.rs | 6 + komorebi-recorder/src/platform/win/apps.rs | 7 + komorebi-recorder/src/platform/win/window.rs | 109 ++++ komorebi-recorder/src/platform/window.rs | 33 ++ komorebi-recorder/src/recorder.rs | 29 ++ komorebi-recorder/src/session.rs | 9 + komorebi-recorder/src/sources/mod.rs | 2 + komorebi-recorder/src/sources/windowtrack.rs | 467 ++++++++++++++++++ komorebi-recorder/src/testkit.rs | 59 +++ komorebi-recorder/tests/video.rs | 91 ++++ 21 files changed, 1293 insertions(+), 2 deletions(-) create mode 100644 komorebi-recorder/src/analysis/window_track.rs create mode 100644 komorebi-recorder/src/sources/windowtrack.rs diff --git a/README.md b/README.md index 8df35910..f0b57bc3 100644 --- a/README.md +++ b/README.md @@ -48,6 +48,8 @@ On macOS the terminal needs the Screen Recording permission, as `komorebi` does. with `--log-level trace` first: the recorder never changes komorebi's log level, it only warns. A session made with `--no-video` has everything but the screen. [Recording a bug repro](docs/usage/recording-a-bug-repro.md) has the rest: what is captured on each OS, docking, sleep and lock, and recovering a recording after a crash. +`record --window-track` also samples where every window is at video rate, and `sheet --windows` draws those rects on +the frames. ## Investigate it diff --git a/docs/PLATFORM.md b/docs/PLATFORM.md index 47af36be..1b80f58b 100644 --- a/docs/PLATFORM.md +++ b/docs/PLATFORM.md @@ -109,6 +109,22 @@ macOS decodes with AVFoundation's `AVAssetReader`, Windows with Media Foundation host with nothing: it says so when asked to open a file. Both real hosts are asked for BGRA frames at a chosen size, so that a contact sheet never decodes more pixels than it shows. +### The window track + +`record --window-track[=HZ]` samples every top-level window into `windows.jsonl` (see +`src/sources/windowtrack.rs`), listed by `platform::window::list`. + +| | macOS | Windows | +|---|---|---| +| source | `CGWindowListCopyWindowInfo(OptionAll)`, layer 0 windows | `EnumWindows` once per tick, then cheap reads of each window | +| `id` | the window server's window number | the window handle (komorebi's `hwnd`) | +| `rect` | `kCGWindowBounds`, in display points | the visible frame (`DWMWA_EXTENDED_FRAME_BOUNDS`), in pixels | +| `visible` | `kCGWindowIsOnscreen` (false for windows on other spaces) | `IsWindowVisible`; invisible windows and tool windows are not listed | +| `minimized` | always false: the window server does not say | `IsIconic` | +| `cloaked` | absent | `DWMWA_CLOAKED` | +| `title` | `kCGWindowName` | `GetWindowTextW` | +| status | written against the `objc2` sources and not built or run | built and run | + ### The program's own windows The recorder window and the viewer must not be in the video they are recording, and komorebi must not tile them. diff --git a/docs/usage/investigating-a-recording.md b/docs/usage/investigating-a-recording.md index b38e45f9..1829867b 100644 --- a/docs/usage/investigating-a-recording.md +++ b/docs/usage/investigating-a-recording.md @@ -118,7 +118,7 @@ and `event:12.end` when it finished. | `logq QUERY` | Query komorebi's log by field, with no escape codes or span prefixes to strip: `level=ERROR and message~ghost`, `hwnd=0x77179C` (decimal and hex are the same window), `command=MoveWindow`; `--count`, `--group-by FIELD`, `--from`/`--to`, `--spans` | | `search TEXT` | Where a word appears: komorebi's log, events, incidents, annotations (and `--oslog` for the macOS trace) | | `changes` | When and where on each display the picture changed, and which events fall inside | -| `sheet` | A contact sheet image of frames with times, events and the bug region marked | +| `sheet` | A contact sheet image of frames with times, events and the bug region marked; `--windows` also draws each window's rect on the frames when the session has a window track | | `motion [--region x,y,w,h]` | When, and how hard, a display or a region of it was moving: segments with start, duration, frame count, peak share of pixels changed and peak mean brightness delta (replaces scripting ffmpeg `tblend`) | | `framediff A B [--region ..] [--png FILE] [--against SESSION]` | How two frames differ: changed share, mean and peak brightness delta, the box of change, and a PNG with the changed pixels in red | | `blank --region x,y,w,h` | When a region was blank (one colour, named: `white`, `black`, a hex code) or frozen (identical to the frame before): "the window went white for 200 ms" | diff --git a/docs/usage/recording-a-bug-repro.md b/docs/usage/recording-a-bug-repro.md index 69806fa8..27029453 100644 --- a/docs/usage/recording-a-bug-repro.md +++ b/docs/usage/recording-a-bug-repro.md @@ -92,6 +92,7 @@ Useful options: | `--oslog-all` | One extra unfiltered system-wide stream (very large) | | `--no-oslog`, `--no-komorebi` | Skip the OS trace, or skip touching komorebi entirely | | `--no-video` | Do not capture the screen; record everything else on the same clock (see below) | +| `--window-track[=HZ]` | Also sample every window's rect and state alongside the video (30 Hz by default, 1 to 120; see below) | | `--no-probe-windows` | Windows: do not record what Windows says about each window event, which is on by default (see below) | | `--fps N`, `--retina`, `--codec hevc\|h264` | Video settings (display-rate capture (`--fps 0`, the default; N caps it), logical points, HEVC by default) | @@ -126,6 +127,19 @@ those facts. `--no-probe-windows` turns it off, and without it `overview` cannot really hidden, nor find a ghost tile. It does nothing on a Mac, where a hidden window is parked in a corner and the state says so, and asking for `--probe-windows` there is an error. +### Tracking where every window is + +The per-event probe above says what one window looked like when something happened. The recurring +question is different: _which window was where, in this frame_. `record --window-track` (or +`--window-track=60`) lists every top-level window many times a second, with its rect, whether it is +visible or minimized and, on Windows, whether it is cloaked, front to back, into `windows.jsonl`. A +row is written only when something changed, plus a full row each second, so a still desktop costs +about one row a second and a busy one a few kilobytes a second. The recorder's own windows and tool +windows (tooltips, toasts) are left out. It is off unless asked for. `komorebi-recorder sheet + --windows` then draws each window's rect, labelled with its program and id, on every +frame of the sheet; a session with no track says so and gets a plain sheet. A recording that cannot +keep the rate says so in its warnings. + ### Checking your setup ``` @@ -223,6 +237,7 @@ frames-[-].csv every frame's host-clock timestamp komorebi.log exact slice of komorebi's own log events.jsonl, states/ subscriber notifications; full State stored only when it changed state-start.json, state-end.json +windows.jsonl window track (only with --window-track): every window's rect, on change and once a second recorder.jsonl what happened to the recording environment (displays, sleep/wake, lock, restarts) annotations.jsonl bug regions, notes and findings (appended, never rewritten) oslog/komorebi.ndjson unified log, komorebi's process (oslog/komorebi-.ndjson after a restart) diff --git a/komorebi-recorder/src/analysis/mod.rs b/komorebi-recorder/src/analysis/mod.rs index cd98e6fa..5ef29200 100644 --- a/komorebi-recorder/src/analysis/mod.rs +++ b/komorebi-recorder/src/analysis/mod.rs @@ -44,5 +44,6 @@ pub mod timespec; pub mod trace; pub mod video; pub mod views; +pub mod window_track; pub use facade::Analysis; diff --git a/komorebi-recorder/src/analysis/sheet.rs b/komorebi-recorder/src/analysis/sheet.rs index e97d0aaf..51c94b9a 100644 --- a/komorebi-recorder/src/analysis/sheet.rs +++ b/komorebi-recorder/src/analysis/sheet.rs @@ -31,6 +31,17 @@ //! 8192 pixels wide and 16384 tall; a sheet that would be larger is refused (see //! [`Analysis::sheet`]). //! +//! # Windows +//! +//! With [`SheetOptions::windows`] (the `--windows` flag), each frame also has the rect of every +//! window drawn on it, from the session's window track (see [`super::window_track`]): a 2 pixel +//! border in a colour taken from the window's id and a label of the owner and id, front windows +//! drawn last. Windows that are minimized or not visible are not drawn; a cloaked window (Windows +//! only: hidden but still "visible") has a thin dim border, so that a window komorebi has hidden +//! can be told from one that is shown. The windows are those of the moment of the picture +//! shown, which is how [`SheetFrame::windows`] names them. [`rect_in_cell`] is the conversion from +//! the display's units (the `frame` of the display in `session.json`) to the pixels of a cell. +//! //! # Example //! //! Needs `ffmpeg` to make a test video, and a host that can decode it, so it is only compiled here; @@ -55,6 +66,7 @@ //! cell_width: 240, //! columns: 3, //! regions: Vec::new(), +//! windows: None, //! out: session.path("sheets/example.png"), //! })?; //! @@ -66,7 +78,9 @@ use super::Analysis; use super::envelope; use super::video::SessionVideo; +use super::window_track::WindowTrack; use crate::av::Scale; +use crate::sources::windowtrack::WindowSample; use crate::timecode; use ab_glyph::Font; use ab_glyph::FontRef; @@ -125,6 +139,8 @@ pub struct SheetOptions { pub columns: usize, /// The bug regions to mark: a frame whose moment is inside one has a red border. pub regions: Vec, + /// The window track to draw on each frame (see the module docs); `None` draws no windows. + pub windows: Option, /// Where to write the PNG. Its parent directories are created, and a file already there is /// replaced. pub out: PathBuf, @@ -153,6 +169,10 @@ pub struct SheetFrame { pub events: Vec, /// The ids of the bug regions it is inside. pub bugs: Vec, + /// The ids of the windows drawn on it (see the module docs), front to back; empty when the + /// sheet draws no windows. + #[serde(skip_serializing_if = "Vec::is_empty")] + pub windows: Vec, } /// A moment of a display that the sheet has no frame for. @@ -293,6 +313,123 @@ impl Canvas { } } +/// Maps a window's `rect` (`[left, top, right, bottom]`, in the units of a display's `frame`) onto +/// a cell that shows that display `cell_width` by `cell_height` pixels, and returns where it is in +/// the cell as `[x, y, width, height]` in pixels from the cell's top left. The part of the window +/// outside the display is cut off, and a window with nothing inside the display is `None`. What is +/// left is at least one pixel each way. +/// +/// # Example +/// +/// ```rust +/// use komorebi_recorder::analysis::sheet::rect_in_cell; +/// +/// // a 1000 by 800 display at (100, 0), drawn 250 by 200: a quarter size +/// let frame = [100.0, 0.0, 1000.0, 800.0]; +/// assert_eq!(rect_in_cell([200, 40, 600, 440], frame, 250, 200), Some([25, 10, 100, 100])); +/// // sticking out of the left edge is cut off; wholly on another display is not drawn +/// assert_eq!(rect_in_cell([0, 0, 300, 80], frame, 250, 200), Some([0, 0, 50, 20])); +/// assert_eq!(rect_in_cell([-900, 0, -100, 80], frame, 250, 200), None); +/// ``` +#[must_use] +pub fn rect_in_cell( + rect: [i32; 4], + frame: [f64; 4], + cell_width: u32, + cell_height: u32, +) -> Option<[u32; 4]> { + let [frame_x, frame_y, frame_width, frame_height] = frame; + if frame_width <= 0.0 || frame_height <= 0.0 || cell_width == 0 || cell_height == 0 { + return None; + } + + let scale_x = f64::from(cell_width) / frame_width; + let scale_y = f64::from(cell_height) / frame_height; + let left = ((f64::from(rect[0]) - frame_x) * scale_x).max(0.0); + let top = ((f64::from(rect[1]) - frame_y) * scale_y).max(0.0); + let right = ((f64::from(rect[2]) - frame_x) * scale_x).min(f64::from(cell_width)); + let bottom = ((f64::from(rect[3]) - frame_y) * scale_y).min(f64::from(cell_height)); + if right <= left || bottom <= top { + return None; + } + + let (x, y) = (left.round() as u32, top.round() as u32); + let width = ((right.round() as u32).saturating_sub(x)).max(1); + let height = ((bottom.round() as u32).saturating_sub(y)).max(1); + Some([x, y, width, height]) +} + +/// A colour for a window, the same every time: its id spread over the colour wheel. +fn window_color(id: u32) -> [u8; 3] { + let hue = f32::from((id.wrapping_mul(2_654_435_761) >> 8) as u16 % 360) / 60.0; + let channel = |offset: f32| -> u8 { + let k = (offset + hue) % 6.0; + (255.0 * (1.0 - k.min(4.0 - k).clamp(0.0, 1.0)) * 0.75 + 64.0) as u8 + }; + [channel(5.0), channel(3.0), channel(1.0)] +} + +impl Canvas { + /// Darken a rectangle, to make text on top of a picture readable. + fn shade(&mut self, x: u32, y: u32, width: u32, height: u32) { + for row in y..y + height { + for column in x..x + width { + self.put(i64::from(column), i64::from(row), BACKGROUND, 0.7); + } + } + } + + /// Draw `windows` (front to back) on the cell at `x`, `y` that is `width` by `height` pixels + /// and shows `frame`, and return the ids of those drawn. + #[allow(clippy::too_many_arguments)] + fn windows( + &mut self, + font: &FontRef, + windows: &[WindowSample], + frame: [f64; 4], + x: u32, + y: u32, + width: u32, + height: u32, + ) -> Vec { + let mut drawn = Vec::new(); + for window in windows.iter().rev() { + if !window.visible || window.minimized { + continue; + } + let Some([left, top, w, h]) = rect_in_cell(window.rect, frame, width, height) else { + continue; + }; + + let cloaked = window.cloaked == Some(true); + let thickness = if cloaked { 1 } else { 2 }.min(w / 2).min(h / 2).max(1); + let color = if cloaked { + DIM + } else { + window_color(window.id) + }; + self.border(x + left, y + top, w, h, thickness, color); + + if w >= 48 && h >= 14 { + let label = truncate(&format!("{} {}", window.owner, window.id), (w / 6) as usize); + let label_width = (label.chars().count() as u32 * 6 + 4).min(w); + self.shade(x + left + thickness, y + top + thickness, label_width, 13); + self.text( + font, + 10.0, + x + left + thickness + 2, + y + top + thickness, + &label, + color, + ); + } + drawn.push(window.id); + } + drawn.reverse(); + drawn + } +} + fn truncate(text: &str, limit: usize) -> String { let single: String = text.split_whitespace().collect::>().join(" "); if single.chars().count() <= limit { @@ -304,6 +441,22 @@ fn truncate(text: &str, limit: usize) -> String { } } +/// The frame (position and size in the display's own units) of `display` at `video_s`. +fn display_frame(analysis: &Analysis, display: u32, video_s: f64) -> Option<[f64; 4]> { + analysis + .manifest + .segment_at(display, video_s) + .map(|entry| entry.result.display.frame) + .or_else(|| { + analysis + .manifest + .displays + .iter() + .find(|entry| entry.result.display.id == display) + .map(|entry| entry.result.display.frame) + }) +} + impl Analysis { /// Draws a contact sheet of the moments in `options` and writes it as a PNG. /// @@ -472,6 +625,26 @@ impl Analysis { }) .map(|region| region.id.clone()) .collect(); + let windows = match ( + &options.windows, + display_frame(self, *display, cell_frame.shown), + ) { + (Some(track), Some(frame)) => track + .at(self.manifest.mach_at_video_seconds(cell_frame.shown)) + .map(|row| { + canvas.windows( + &font, + &row.windows, + frame, + x, + y, + cell_frame.width, + cell_frame.height, + ) + }) + .unwrap_or_default(), + _ => Vec::new(), + }; if !bugs.is_empty() { canvas.border(x, y, cell_frame.width, cell_frame.height, 3, BUG); } @@ -539,6 +712,7 @@ impl Analysis { height: cell_frame.height, events, bugs, + windows, }); } y += row_height + LABEL_HEIGHT + MARGIN; diff --git a/komorebi-recorder/src/analysis/window_track.rs b/komorebi-recorder/src/analysis/window_track.rs new file mode 100644 index 00000000..11e3c542 --- /dev/null +++ b/komorebi-recorder/src/analysis/window_track.rs @@ -0,0 +1,129 @@ +//! Reading the window track back: which window was where at a moment. +//! +//! A recording made with `--window-track` has `windows.jsonl` (see +//! [`crate::sources::windowtrack`]): rows that each list every window, written when something +//! changed and once a second regardless. [`Analysis::window_track`] reads it into a +//! [`WindowTrack`], and [`WindowTrack::at`] answers "the windows at this `mach_ns`" with the last +//! row at or before it. Because every row is complete there is nothing to replay. This is the one +//! entry point for anything that wants to know where the windows were in a frame: the `sheet +//! --windows` overlay uses it, and a later `--window` filter for `motion` or `blank` would too. +//! +//! A moment before the first row has no answer (`None`): the track had not started. A moment long +//! after the last row gets the last row, since the sampler writes a keyframe every second while it +//! runs; use [`WindowTrack::last_mach_ns`] to tell if that is stale. +//! +//! # Example +//! +//! ```rust +//! use komorebi_recorder::analysis::Analysis; +//! use komorebi_recorder::sources::windowtrack::WINDOWS_FILE; +//! use komorebi_recorder::testkit::{SessionBuilder, mach_ns}; +//! +//! let (session, _) = SessionBuilder::new("doc-window-track") +//! .window_track(&[ +//! (1.0, &[(7, "term", [0, 0, 500, 400])]), +//! (2.0, &[(7, "term", [100, 0, 600, 400])]), +//! ]) +//! .build(); +//! assert!(session.path(WINDOWS_FILE).exists()); +//! +//! let analysis = Analysis::open(&session.dir)?; +//! let track = analysis.window_track()?.expect("the session has a track"); +//! assert!(track.at(mach_ns(0.5)).is_none()); +//! assert_eq!(track.at(mach_ns(1.5)).unwrap().windows[0].rect[0], 0); +//! assert_eq!(track.at(mach_ns(2.5)).unwrap().windows[0].rect[0], 100); +//! # Ok::<(), Box>(()) +//! ``` + +use super::Analysis; +use crate::sources::windowtrack::TrackRow; +use crate::sources::windowtrack::WINDOWS_FILE; +use color_eyre::eyre; +use color_eyre::eyre::WrapErr; + +/// The rows of `windows.jsonl`, in time order. +#[derive(Debug, Clone, Default)] +pub struct WindowTrack { + rows: Vec, +} + +impl WindowTrack { + /// Reads the text of `windows.jsonl`. A line that is not a row (a half-written last line of a + /// recording that was killed) is skipped, and rows are sorted by time. + #[must_use] + pub fn parse(text: &str) -> Self { + let mut rows: Vec = text + .lines() + .filter_map(|line| serde_json::from_str(line).ok()) + .collect(); + rows.sort_by_key(|row| row.mach_ns); + Self { rows } + } + + /// Returns the last row at or before `mach_ns`: the windows as they were then. `None` before + /// the first row. + #[must_use] + pub fn at(&self, mach_ns: u64) -> Option<&TrackRow> { + let after = self.rows.partition_point(|row| row.mach_ns <= mach_ns); + after.checked_sub(1).map(|index| &self.rows[index]) + } + + /// How many rows there are. + #[must_use] + pub fn len(&self) -> usize { + self.rows.len() + } + + /// Whether there are no rows. + #[must_use] + pub fn is_empty(&self) -> bool { + self.rows.is_empty() + } + + /// When the last row was written, on the master clock. + #[must_use] + pub fn last_mach_ns(&self) -> Option { + self.rows.last().map(|row| row.mach_ns) + } +} + +impl Analysis { + /// Reads the session's window track, or `None` if it has none (it was recorded without + /// `--window-track`). The file is read each time this is called, not kept. A session whose + /// recorder died still has what was flushed. + /// + /// # Errors + /// + /// The file exists and cannot be read. + /// + /// # Example + /// + /// See the [module docs](self). + pub fn window_track(&self) -> eyre::Result> { + let path = self.guard().resolve(WINDOWS_FILE)?; + if !path.exists() { + return Ok(None); + } + let text = std::fs::read_to_string(&path) + .wrap_err_with(|| format!("could not read {}", path.display()))?; + Ok(Some(WindowTrack::parse(&text))) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn the_last_row_at_or_before_a_moment_answers() { + let track = WindowTrack::parse( + "{\"mach_ns\":20,\"windows\":[]}\nnot a row\n{\"mach_ns\":10,\"windows\":[]}\n{\"mach_ns\":30", + ); + assert_eq!(track.len(), 2); + assert!(track.at(9).is_none()); + assert_eq!(track.at(10).unwrap().mach_ns, 10); + assert_eq!(track.at(19).unwrap().mach_ns, 10); + assert_eq!(track.at(u64::MAX).unwrap().mach_ns, 20); + assert_eq!(track.last_mach_ns(), Some(20)); + } +} diff --git a/komorebi-recorder/src/cli/record.rs b/komorebi-recorder/src/cli/record.rs index dd6d8d1e..616f9e3a 100644 --- a/komorebi-recorder/src/cli/record.rs +++ b/komorebi-recorder/src/cli/record.rs @@ -76,6 +76,19 @@ pub struct RecordArgs { /// Windows has hidden, such as a ghost tile #[arg(long)] pub no_probe_windows: bool, + /// Also sample every top-level window's rect, visibility and minimized (and, on Windows, + /// cloaked) state at HZ samples a second [default when given without a value: 30; 1 to 120] + /// into `windows.jsonl`, so that "which window was where in this frame" has an answer for + /// every frame; `sheet --windows` draws it. Written only when something changed, plus a full + /// row each second, so a still desktop costs almost nothing. Off unless asked for + #[arg( + long, + value_name = "HZ", + num_args = 0..=1, + require_equals = true, + default_missing_value = "30" + )] + pub window_track: Option, /// Do not capture the screen: record everything else (komorebi's events and log, the OS /// trace and incidents) on the same clock #[arg(long, conflicts_with = "displays")] @@ -122,6 +135,7 @@ impl Default for RecordArgs { no_video: false, probe_windows: false, no_probe_windows: false, + window_track: None, no_oslog: false, oslog_apps: AppsMode::Managed, oslog_all: false, @@ -187,6 +201,7 @@ impl RecordArgs { no_komorebi: self.no_komorebi, no_video: self.no_video, probe_windows: self.probes_windows(), + window_track: self.window_track, } } diff --git a/komorebi-recorder/src/cli/video.rs b/komorebi-recorder/src/cli/video.rs index c6224ba1..f530f8d3 100644 --- a/komorebi-recorder/src/cli/video.rs +++ b/komorebi-recorder/src/cli/video.rs @@ -103,6 +103,10 @@ pub struct SheetArgs { /// Frames per row #[arg(long, default_value_t = 4)] pub columns: usize, + /// Draw each window's rect on every frame, from the window track (`record --window-track`); + /// a session without one says so and gets a plain sheet + #[arg(long)] + pub windows: bool, /// Where to write the PNG [default: in the session's sheets/ directory, named by the range] #[arg(long, value_name = "FILE")] pub out: Option, @@ -326,6 +330,21 @@ pub(super) fn sheet(args: &SheetArgs) -> eyre::Result<()> { )) }); + let mut warnings = analysis.warnings(); + let windows = if args.windows { + let track = analysis.window_track()?; + if track.is_none() { + warnings.push( + "--windows was asked for but the session has no window track (it was recorded \ + without --window-track), so no windows are drawn" + .to_string(), + ); + } + track + } else { + None + }; + let data = analysis.sheet( &video, &SheetOptions { @@ -334,13 +353,17 @@ pub(super) fn sheet(args: &SheetArgs) -> eyre::Result<()> { cell_width: args.width, columns: args.columns, regions, + windows, out, }, )?; if args.json { - return print_json(&Envelope::ok(SCHEMA_SHEET, data).warnings(analysis.warnings())); + return print_json(&Envelope::ok(SCHEMA_SHEET, data).warnings(warnings)); } + for warning in &warnings { + eprintln!("note: {warning}"); + } print!("{}", report::sheet_text(&data)); Ok(()) } diff --git a/komorebi-recorder/src/guide.md b/komorebi-recorder/src/guide.md index 8823ebf1..b84cd72a 100644 --- a/komorebi-recorder/src/guide.md +++ b/komorebi-recorder/src/guide.md @@ -101,6 +101,11 @@ locations as hints. `blank --region x,y,w,h` (when a region went one colour, such as white, or froze) and `compare ` (two recordings' frames, motion and findings side by side). Regions are in display points, the same as the `bbox_points` of `changes`. They need a host that can decode the video. +- A session recorded with `--window-track` has `windows.jsonl`: every top-level window's rect, + visibility and minimized state (and `cloaked` on Windows) sampled at video rate, front to back, + so "which window was where in this frame" has an answer. `sheet --windows` draws the rects on + each frame (`windows` in the answer names the ones drawn). Rects are in the display's own units + (pixels on Windows, points on a Mac). Without it only the sparse per-event probes exist. - Video time `00:00.000` is the first frame of any display. A display that was unplugged, asleep or whose capture failed has a gap, and `incident` records say why. diff --git a/komorebi-recorder/src/platform/macos/apps.rs b/komorebi-recorder/src/platform/macos/apps.rs index c5a62530..2e1fb8dd 100644 --- a/komorebi-recorder/src/platform/macos/apps.rs +++ b/komorebi-recorder/src/platform/macos/apps.rs @@ -8,14 +8,19 @@ //! window handle; see [`window_owners`] for how they differ. use crate::sources::managed_apps::WindowOwner; +use crate::sources::windowtrack::WindowSample; use objc2_core_foundation::CFArray; +use objc2_core_foundation::CFBoolean; use objc2_core_foundation::CFDictionary; use objc2_core_foundation::CFNumber; use objc2_core_foundation::CFString; use objc2_core_graphics::CGWindowListCopyWindowInfo; use objc2_core_graphics::CGWindowListOption; use objc2_core_graphics::kCGNullWindowID; +use objc2_core_graphics::kCGWindowBounds; +use objc2_core_graphics::kCGWindowIsOnscreen; use objc2_core_graphics::kCGWindowLayer; +use objc2_core_graphics::kCGWindowName; use objc2_core_graphics::kCGWindowNumber; use objc2_core_graphics::kCGWindowOwnerName; use objc2_core_graphics::kCGWindowOwnerPID; @@ -93,3 +98,97 @@ pub fn window_owners() -> HashMap { owners } + +/// Reads one number out of a `kCGWindowBounds` dictionary (`X`, `Y`, `Width` or `Height`). +fn bounds_value(bounds: &CFDictionary, key: &str) -> Option { + let key = CFString::from_str(key); + // SAFETY: the values of a bounds dictionary are CFNumbers, per the CGWindow documentation + unsafe { dictionary_value::(bounds, &key).and_then(|n| n.as_ref().as_f64()) } +} + +/// Returns the ordinary windows (layer 0, with a frame) the window server lists, front to back, +/// as [`WindowSample`]s, without this process's own windows. `rect` is `kCGWindowBounds` in +/// display points rounded to whole points, `visible` is `kCGWindowIsOnscreen`, and `cloaked` is +/// `None` because a Mac has no such thing. This is how [`crate::platform::window::list`] is +/// answered here. +/// +/// Not built or run when this was written: it was checked against the `objc2` sources only. +#[must_use] +pub fn window_list() -> Vec { + let mut windows = Vec::new(); + let Some(list) = CGWindowListCopyWindowInfo(CGWindowListOption::OptionAll, kCGNullWindowID) + else { + return windows; + }; + + let own = std::process::id() as i32; + let list: &CFArray = &list; + for index in 0..list.count() { + // SAFETY: every element of the window list is a CFDictionary, and `index` is in bounds + let Some(dictionary) = + (unsafe { list.value_at_index(index).cast::().as_ref() }) + else { + continue; + }; + + // SAFETY: values under these keys have these types per the CGWindow documentation + let (number, pid, owner, title, layer, onscreen, bounds) = unsafe { + ( + dictionary_value::(dictionary, kCGWindowNumber) + .and_then(|number| number.as_ref().as_i64()), + dictionary_value::(dictionary, kCGWindowOwnerPID) + .and_then(|pid| pid.as_ref().as_i32()), + dictionary_value::(dictionary, kCGWindowOwnerName) + .map(|name| name.as_ref().to_string()), + dictionary_value::(dictionary, kCGWindowName) + .map(|name| name.as_ref().to_string()), + dictionary_value::(dictionary, kCGWindowLayer) + .and_then(|layer| layer.as_ref().as_i64()), + dictionary_value::(dictionary, kCGWindowIsOnscreen) + .map(|onscreen| onscreen.as_ref().as_bool()), + dictionary_value::(dictionary, kCGWindowBounds), + ) + }; + + let (Some(number), Some(pid), Some(bounds)) = (number, pid, bounds) else { + continue; + }; + if pid == own || layer.unwrap_or_default() != 0 { + continue; + } + let Ok(id) = u32::try_from(number) else { + continue; + }; + + // SAFETY: `bounds` came out of the live window dictionary above + let bounds = unsafe { bounds.as_ref() }; + let (Some(x), Some(y), Some(width), Some(height)) = ( + bounds_value(bounds, "X"), + bounds_value(bounds, "Y"), + bounds_value(bounds, "Width"), + bounds_value(bounds, "Height"), + ) else { + continue; + }; + if width < 1.0 || height < 1.0 { + continue; + } + + windows.push(WindowSample { + id, + owner: owner.unwrap_or_default(), + title: title.filter(|title| !title.is_empty()), + rect: [ + x.round() as i32, + y.round() as i32, + (x + width).round() as i32, + (y + height).round() as i32, + ], + visible: onscreen.unwrap_or(false), + minimized: false, + cloaked: None, + }); + } + + windows +} diff --git a/komorebi-recorder/src/platform/unsupported.rs b/komorebi-recorder/src/platform/unsupported.rs index 4048758c..43546f6d 100644 --- a/komorebi-recorder/src/platform/unsupported.rs +++ b/komorebi-recorder/src/platform/unsupported.rs @@ -14,6 +14,12 @@ pub mod apps { pub fn window_owners() -> HashMap { HashMap::new() } + + /// Returns no windows: this host has no window server to ask. + #[must_use] + pub fn window_list() -> Vec { + Vec::new() + } } /// Screen capture, which this host cannot do. diff --git a/komorebi-recorder/src/platform/win/apps.rs b/komorebi-recorder/src/platform/win/apps.rs index bc170c9b..7f1e18fd 100644 --- a/komorebi-recorder/src/platform/win/apps.rs +++ b/komorebi-recorder/src/platform/win/apps.rs @@ -100,3 +100,10 @@ pub fn window_owners() -> HashMap { }) .collect() } + +/// Returns the top-level windows a user could see, front to back (see +/// [`crate::platform::window::list`], which is how the rest of the crate reaches this). +#[must_use] +pub fn window_list() -> Vec { + super::window::list() +} diff --git a/komorebi-recorder/src/platform/win/window.rs b/komorebi-recorder/src/platform/win/window.rs index 3f8ce4ed..a494f0ab 100644 --- a/komorebi-recorder/src/platform/win/window.rs +++ b/komorebi-recorder/src/platform/win/window.rs @@ -11,12 +11,16 @@ //! | [`exclude_from_capture`] | asking for this process's windows to be left out of captures | //! | [`own_window_handles`] | the handles of those windows, as komorebi knows windows | //! | [`inspect`] | what Windows says about any window, for the per-event probe | +//! | [`list`] | every window an ordinary user sees, for the sampled window track | //! //! This module is not part of the facade: [`crate::platform::window`], which is shared by every //! host, calls it. Nothing tests it automatically; it acts on real windows, and is checked on a //! real machine (see `docs/dev/windows-checkpoints.md`). +use crate::sources::windowtrack::WindowSample; use crate::sources::winprobe::WindowFacts; +use std::cell::RefCell; +use std::collections::HashMap; use std::ffi::c_void; use windows::Win32::Foundation::HWND; use windows::Win32::Foundation::LPARAM; @@ -26,13 +30,17 @@ use windows::Win32::Graphics::Dwm::DWMWA_EXTENDED_FRAME_BOUNDS; use windows::Win32::Graphics::Dwm::DwmGetWindowAttribute; use windows::Win32::System::Threading::GetCurrentProcessId; use windows::Win32::UI::WindowsAndMessaging::EnumWindows; +use windows::Win32::UI::WindowsAndMessaging::GWL_EXSTYLE; +use windows::Win32::UI::WindowsAndMessaging::GetWindowLongW; use windows::Win32::UI::WindowsAndMessaging::GetWindowRect; +use windows::Win32::UI::WindowsAndMessaging::GetWindowTextW; use windows::Win32::UI::WindowsAndMessaging::GetWindowThreadProcessId; use windows::Win32::UI::WindowsAndMessaging::IsIconic; use windows::Win32::UI::WindowsAndMessaging::IsWindow; use windows::Win32::UI::WindowsAndMessaging::IsWindowVisible; use windows::Win32::UI::WindowsAndMessaging::SetWindowDisplayAffinity; use windows::Win32::UI::WindowsAndMessaging::WDA_EXCLUDEFROMCAPTURE; +use windows::Win32::UI::WindowsAndMessaging::WS_EX_TOOLWINDOW; use windows::core::BOOL; unsafe extern "system" fn collect(window: HWND, found: LPARAM) -> BOOL { @@ -142,3 +150,104 @@ pub fn inspect(hwnd: u32) -> Option { Some(facts) } + +unsafe extern "system" fn collect_all(window: HWND, found: LPARAM) -> BOOL { + // SAFETY: `found` is the address of the vector `list` passed to `EnumWindows` + let found = unsafe { &mut *(found.0 as *mut Vec) }; + found.push(window); + BOOL(1) +} + +thread_local! { + /// Program names by process id, so that sampling many times a second does not open every + /// window's process every time. Emptied when it grows large, which also forgets reused ids. + static PROGRAMS: RefCell> = RefCell::new(HashMap::new()); +} + +fn program_of(pid: u32) -> String { + PROGRAMS.with(|programs| { + let mut programs = programs.borrow_mut(); + if programs.len() > 512 { + programs.clear(); + } + programs + .entry(pid) + .or_insert_with(|| super::apps::process_name(pid).unwrap_or_default()) + .clone() + }) +} + +/// Returns the top-level windows a user could see, front to back, as [`WindowSample`]s: one +/// `EnumWindows` pass, then a few cheap reads of each window. +/// +/// Left out: windows that are not visible (`IsWindowVisible`), tool windows (`WS_EX_TOOLWINDOW`: +/// tooltips, toasts, floating palettes), windows with an empty frame, and this process's own +/// windows (the recorder and the viewer are not in the video either). The rect is the visible +/// frame, as [`inspect`] reads it. A cloaked window (another virtual desktop, or hidden by +/// komorebi) is listed with `cloaked` true. The program name is looked up once for each process +/// and kept for the thread's life. It is reached through [`crate::platform::window::list`]. +#[must_use] +pub fn list() -> Vec { + let mut handles: Vec = Vec::new(); + // SAFETY: `handles` outlives the call and `collect_all` only touches it during it + let _ = unsafe { EnumWindows(Some(collect_all), LPARAM(&raw mut handles as isize)) }; + // SAFETY: no preconditions + let own = unsafe { GetCurrentProcessId() }; + + let mut samples = Vec::new(); + for window in handles { + // SAFETY: queries about a handle that may have gone since it was listed, which they + // report as a failure; every out pointer is valid for its call + unsafe { + if !IsWindowVisible(window).as_bool() { + continue; + } + if GetWindowLongW(window, GWL_EXSTYLE) as u32 & WS_EX_TOOLWINDOW.0 != 0 { + continue; + } + let mut pid = 0_u32; + GetWindowThreadProcessId(window, Some(&raw mut pid)); + if pid == 0 || pid == own { + continue; + } + + let mut frame = RECT::default(); + let framed = DwmGetWindowAttribute( + window, + DWMWA_EXTENDED_FRAME_BOUNDS, + (&raw mut frame).cast(), + size_of::() as u32, + ) + .is_ok(); + if !framed && GetWindowRect(window, &raw mut frame).is_err() { + continue; + } + if frame.right <= frame.left || frame.bottom <= frame.top { + continue; + } + + let mut cloaked = 0_u32; + let _ = DwmGetWindowAttribute( + window, + DWMWA_CLOAKED, + (&raw mut cloaked).cast(), + size_of::() as u32, + ); + + let mut buffer = [0_u16; 256]; + let length = GetWindowTextW(window, &mut buffer).max(0) as usize; + let title = String::from_utf16_lossy(&buffer[..length.min(buffer.len())]); + + samples.push(WindowSample { + id: window.0 as usize as u32, + owner: program_of(pid), + title: (!title.is_empty()).then_some(title), + rect: [frame.left, frame.top, frame.right, frame.bottom], + visible: true, + minimized: IsIconic(window).as_bool(), + cloaked: Some(cloaked != 0), + }); + } + } + samples +} diff --git a/komorebi-recorder/src/platform/window.rs b/komorebi-recorder/src/platform/window.rs index b9598e87..95354038 100644 --- a/komorebi-recorder/src/platform/window.rs +++ b/komorebi-recorder/src/platform/window.rs @@ -16,6 +16,7 @@ //! per-event probe (`crate::sources::winprobe`), and is `None` on every other host. use crate::komorebi_link; +use crate::sources::windowtrack::WindowSample; use crate::sources::winprobe::WindowFacts; use std::time::Duration; use std::time::Instant; @@ -66,6 +67,38 @@ pub fn inspect(hwnd: u32) -> Option { } } +/// Whether this host can list its windows (see [`list`]). Where it cannot, `--window-track` is +/// refused rather than recording nothing. +pub const CAN_LIST: bool = cfg!(any(windows, target_os = "macos")); + +/// Returns every top-level window an ordinary user could see, front to back, as generic facts +/// for the sampled window track (`crate::sources::windowtrack`). The recorder's own windows are +/// left out. +/// +/// | | Windows | macOS | +/// |---|---|---| +/// | `id` | the window handle, as komorebi's `hwnd` | the window server's window number | +/// | `rect` | visible frame in pixels | `kCGWindowBounds` in display points | +/// | `visible` | `IsWindowVisible`, always true here | `kCGWindowIsOnscreen` | +/// | `cloaked` | `Some(_)` from DWM | `None` | +/// +/// It is empty on a host that cannot list windows ([`CAN_LIST`]). +/// +/// # Example +/// +/// ```rust +/// use komorebi_recorder::platform::window::{CAN_LIST, list}; +/// +/// // without windows to ask, there is nothing listed (a desktop session lists something) +/// if !CAN_LIST { +/// assert!(list().is_empty()); +/// } +/// ``` +#[must_use] +pub fn list() -> Vec { + super::apps::window_list() +} + /// Per-window housekeeping for a window's update loop. Call [`tick`](Chrome::tick) on every frame: /// it is cheap, and does something about once a second. /// diff --git a/komorebi-recorder/src/recorder.rs b/komorebi-recorder/src/recorder.rs index 61223e89..ba004ef2 100644 --- a/komorebi-recorder/src/recorder.rs +++ b/komorebi-recorder/src/recorder.rs @@ -52,6 +52,7 @@ use crate::sources::oslog; use crate::sources::oslog::OsLogLevel; use crate::sources::oslog::StreamLive; use crate::sources::oslog::StreamSet; +use crate::sources::windowtrack::WindowTracker; use crate::sources::winprobe::EventProber; use crate::sources::wintrace::TraceSet; use crate::system_state; @@ -171,6 +172,9 @@ pub struct RecordOptions { /// cloaked, minimized, where) and, now and then, ask komorebi whether it manages it. komorebi /// for Windows hides windows by cloaking them in place, which `State` cannot show. pub probe_windows: bool, + /// Sample every window's rect at this many hertz into `windows.jsonl` (see + /// `crate::sources::windowtrack`). `None` records no window track. + pub window_track: Option, } /// Where a recording is in its life. @@ -708,6 +712,7 @@ struct Recording<'a> { apps: AppStreams, tailer: Option, events: Option, + window_track: Option, discovery_rx: Receiver, displays: DisplayManager, komorebi: KomorebiWatch, @@ -973,6 +978,24 @@ impl<'a> Recording<'a> { } }; + let window_track = match options.window_track { + None => None, + Some(_) if !crate::platform::window::CAN_LIST => { + warnings.push("the window track is not available on this host".to_string()); + None + } + Some(hz) => match WindowTracker::start(&session.dir, hz) { + Ok(tracker) => { + manifest.window_track = Some(tracker.initial_info()); + Some(tracker) + } + Err(error) => { + warnings.push(format!("the window track is not being recorded: {error:#}")); + None + } + }, + }; + let tailer = if options.no_komorebi { None } else { @@ -1044,6 +1067,7 @@ impl<'a> Recording<'a> { apps, tailer, events, + window_track, discovery_rx, displays, komorebi: KomorebiWatch::new(!options.no_komorebi, komorebi_process.as_ref()), @@ -1436,6 +1460,7 @@ impl<'a> Recording<'a> { .take() .map(EventRecorder::stop) .unwrap_or_default(); + let window_track = self.window_track.take().map(WindowTracker::stop); let mut stream_infos = self.apps.streams.stop(); if let Some(trace) = &mut self.apps.trace { stream_infos.extend(trace.stop()); @@ -1481,6 +1506,10 @@ impl<'a> Recording<'a> { } self.manifest.events = event_summary; + if let Some(info) = window_track { + warnings.extend(info.warning()); + self.manifest.window_track = Some(info); + } self.manifest.apps = self.app_records(&stream_infos); for window in self.apps.tracker.unresolved() { diff --git a/komorebi-recorder/src/session.rs b/komorebi-recorder/src/session.rs index 7ef56acd..ea49c88b 100644 --- a/komorebi-recorder/src/session.rs +++ b/komorebi-recorder/src/session.rs @@ -14,6 +14,7 @@ use crate::sources::komorebi_log::LogSegment; use crate::sources::komorebi_version::KomorebiVersion; use crate::sources::managed_apps::AppRecord; use crate::sources::oslog::StreamInfo; +use crate::sources::windowtrack::WindowTrackInfo; use chrono::DateTime; use chrono::Utc; use color_eyre::eyre; @@ -212,6 +213,9 @@ pub struct Manifest { pub apps: Vec, /// What was recorded of komorebi's subscriber events. pub events: EventSummary, + /// The sampled window track (`windows.jsonl`), if `--window-track` was asked for. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub window_track: Option, /// `host-clock` when frame timestamps were verified to be on the mach clock. pub sync_quality: String, /// The result of `selftest`, if it was run on this session. @@ -247,6 +251,7 @@ impl Manifest { oslog: Vec::new(), apps: Vec::new(), events: EventSummary::default(), + window_track: None, sync_quality: "host-clock".to_string(), calibration: None, warnings: Vec::new(), @@ -540,6 +545,10 @@ pub fn readme(dir: &Path, manifest: &Manifest) -> String { out.push_str( "| `state-start.json`, `state-end.json` | `State` when recording started / stopped |\n", ); + if manifest.window_track.is_some() { + out.push_str("| `windows.jsonl` | The window track: every top-level window's rect and state, front to back, written when something changed and in full once a second: `{mach_ns, key?, windows: [{id, owner, title?, rect: [l,t,r,b], visible, minimized, cloaked?}]}`. `sheet --windows` draws it on frames | +"); + } out.push_str(os_trace_rows(&host)); out.push_str("| `timeline.jsonl` | Merged index of every record, sorted by time, pointing into the raw files |\n\n"); diff --git a/komorebi-recorder/src/sources/mod.rs b/komorebi-recorder/src/sources/mod.rs index e3f52c10..7ef58ee7 100644 --- a/komorebi-recorder/src/sources/mod.rs +++ b/komorebi-recorder/src/sources/mod.rs @@ -14,6 +14,7 @@ //! | `oslog` | `oslog/*.ndjson`, one `log stream` per seam (komorebi, the system, each managed app) | macOS | //! | `wintrace` | `oslog/system.ndjson` (`WinEvents`) and `oslog/eventlog.ndjson` (Event Log) | Windows | //! | `winprobe` | no file of its own: a `probe` key on rows of `events.jsonl` | Windows, with `--probe-windows` | +//! | `windowtrack` | `windows.jsonl`: every window's rect, sampled at video rate | both hosts, with `--window-track` | //! | `komorebi_version` | no file: asks komorebi's binary what it is, for the manifest | both hosts | //! | `managed_apps` | no file: decides which processes get a trace of their own | macOS | //! @@ -42,5 +43,6 @@ pub mod komorebi_log; pub mod komorebi_version; pub mod managed_apps; pub mod oslog; +pub mod windowtrack; pub mod winprobe; pub mod wintrace; diff --git a/komorebi-recorder/src/sources/windowtrack.rs b/komorebi-recorder/src/sources/windowtrack.rs new file mode 100644 index 00000000..0a0d0415 --- /dev/null +++ b/komorebi-recorder/src/sources/windowtrack.rs @@ -0,0 +1,467 @@ +//! The window track: where every top-level window was, sampled at video rate. +//! +//! The recurring question about a recording is "which window was where, in *this* frame". The +//! other sources cannot say: komorebi's `State` is what komorebi believes and arrives at event +//! times, and the per-event probe (`winprobe`) reads one window at the moment of one event. This +//! source asks the OS for every window's rect, many times a second, so any frame has an answer. +//! +//! It is off unless asked for (`record --window-track[=HZ]`, 30 Hz when no rate is given, clamped +//! to [`MIN_HZ`]..=[`MAX_HZ`]), and writes `windows.jsonl` in the session directory. +//! +//! # The file +//! +//! One [`TrackRow`] per line. A row is the **whole** list of windows at an instant, front to +//! back, so a reader never has to replay changes: the state at a time is the last row at or +//! before it (see `Analysis::window_track` for that). A row is written only when something +//! changed since the previous sample, plus a full row marked `"key":true` once a second +//! ([`KEYFRAME_INTERVAL_NS`]), so a still desktop costs about one row a second. +//! +//! ```json +//! {"mach_ns":5000000000,"key":true,"windows":[{"id":131072,"owner":"chrome","title":"Docs","rect":[0,0,960,1040],"visible":true,"minimized":false,"cloaked":false}]} +//! ``` +//! +//! What counts as a change is any difference in which windows there are, their order (front to +//! back), rect, visibility, minimized or cloaked state. A window's *title* is not compared, since +//! browsers and terminals change theirs constantly; the title in a row is the current one when the +//! row was written, so it can be a second old on a row that was only a keyframe. +//! +//! Rects are in the host's own units: pixels in the virtual screen on Windows, display points on +//! macOS. Both are the units of a display's `frame` in `session.json`, which is how the `sheet +//! --windows` overlay places them on a frame. `cloaked` is `null` or absent on macOS. +//! +//! # Threads and cost +//! +//! A [`WindowTracker`] owns one thread (`window-track`) that owns the file, as the event recorder's +//! writer does. It enumerates windows once per tick, compares with the previous sample, and +//! appends and flushes at most once a second. If a tick is late by more than a whole interval the +//! sampler does not try to catch up, and counts it in [`WindowTrackInfo::late_ticks`]; the recorder +//! warns when that is common. +//! +//! # Example +//! +//! Change detection is plain data and needs no OS: +//! +//! ```rust +//! use komorebi_recorder::sources::windowtrack::{ChangeFilter, WindowSample}; +//! +//! let window = |left| WindowSample { +//! id: 7, +//! owner: "term".to_string(), +//! title: None, +//! rect: [left, 0, left + 100, 100], +//! visible: true, +//! minimized: false, +//! cloaked: Some(false), +//! }; +//! +//! let mut filter = ChangeFilter::default(); +//! // the first sample is always written, a still desktop is not, a move is +//! assert!(filter.observe(0, vec![window(0)]).is_some()); +//! assert!(filter.observe(33_000_000, vec![window(0)]).is_none()); +//! assert!(filter.observe(66_000_000, vec![window(5)]).is_some()); +//! // and a second later it is written again as a keyframe +//! assert!(filter.observe(1_100_000_000, vec![window(5)]).is_some_and(|row| row.key)); +//! ``` + +use crate::clock; +use color_eyre::eyre; +use color_eyre::eyre::WrapErr; +use serde::Deserialize; +use serde::Serialize; +use std::fs::File; +use std::io::BufWriter; +use std::io::Write; +use std::path::Path; +use std::sync::Arc; +use std::sync::atomic::AtomicBool; +use std::sync::atomic::Ordering; +use std::thread::JoinHandle; +use std::time::Duration; +use std::time::Instant; + +/// The file the track is written to, in the session directory. +pub const WINDOWS_FILE: &str = "windows.jsonl"; + +/// The rate used for `--window-track` given without one. +pub const DEFAULT_HZ: u32 = 30; +/// The slowest rate that can be asked for. +pub const MIN_HZ: u32 = 1; +/// The fastest rate that can be asked for. +pub const MAX_HZ: u32 = 120; + +/// How long a row can go without being repeated in full, in nanoseconds. +pub const KEYFRAME_INTERVAL_NS: u64 = 1_000_000_000; + +/// One window at one instant. See the module docs for the units of `rect`. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct WindowSample { + /// The window's id: the handle on Windows (komorebi's `hwnd`), the window number on a Mac. + pub id: u32, + /// The program that owns it: its file name without `.exe` on Windows, the owner name on a Mac. + /// Empty if it could not be found out. + pub owner: String, + /// The window's title, if it has one and the host would say. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub title: Option, + /// Left, top, right and bottom edges. For a minimized window on Windows these are the + /// parked coordinates Windows reports, which are far off screen. + pub rect: [i32; 4], + /// Whether the host says the window is shown (on screen, on a Mac). + pub visible: bool, + /// Whether the window is minimized. Always false on a Mac, which does not say. + pub minimized: bool, + /// Whether the window is cloaked (hidden by the shell, its application, or komorebi, while + /// still "visible"). `None` where the host has no such thing (a Mac). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub cloaked: Option, +} + +impl WindowSample { + /// Whether `other` is this window in the same place and state; titles are not compared. + #[must_use] + pub fn same_as(&self, other: &Self) -> bool { + self.id == other.id + && self.owner == other.owner + && self.rect == other.rect + && self.visible == other.visible + && self.minimized == other.minimized + && self.cloaked == other.cloaked + } +} + +/// One line of `windows.jsonl`: every window at one instant, front to back. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct TrackRow { + /// When the windows were listed, in nanoseconds on the host's master clock. + pub mach_ns: u64, + /// Whether the row is there only because a second passed, not because anything changed. + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + pub key: bool, + /// The windows, front to back. + pub windows: Vec, +} + +/// Decides which samples become rows: the first, any that differ from the previous sample, and one +/// per [`KEYFRAME_INTERVAL_NS`] otherwise. See the module docs for an example. +#[derive(Debug, Default)] +pub struct ChangeFilter { + previous: Vec, + last_written_ns: Option, +} + +impl ChangeFilter { + /// Returns the row to write for a sample taken at `mach_ns`, or `None` if nothing changed and + /// no keyframe is due. A sample is compared with the previous *sample*, not the previous row. + pub fn observe(&mut self, mach_ns: u64, windows: Vec) -> Option { + let changed = self.last_written_ns.is_none() + || windows.len() != self.previous.len() + || windows + .iter() + .zip(&self.previous) + .any(|(now, before)| !now.same_as(before)); + let due = self + .last_written_ns + .is_none_or(|at| mach_ns.saturating_sub(at) >= KEYFRAME_INTERVAL_NS); + + self.previous.clone_from(&windows); + if changed || due { + self.last_written_ns = Some(mach_ns); + Some(TrackRow { + mach_ns, + key: !changed, + windows, + }) + } else { + None + } + } +} + +/// What a recording says about its window track, in `session.json` as `window_track`. +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct WindowTrackInfo { + /// The file, relative to the session directory. + pub file: String, + /// The rate asked for, in samples a second. + pub hz: u32, + /// How many rows were written. + #[serde(default)] + pub rows: u64, + /// How many of those were keyframes only. + #[serde(default)] + pub keyframes: u64, + /// How many samples were taken. + #[serde(default)] + pub ticks: u64, + /// How many samples came more than a whole interval after they were due. + #[serde(default)] + pub late_ticks: u64, + /// Why the sampler stopped early, if it did. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub error: Option, +} + +impl WindowTrackInfo { + /// What the reader of the recording should be told about this track, if anything: it stopped + /// early, or more than one sample in ten was late. + #[must_use] + pub fn warning(&self) -> Option { + if let Some(error) = &self.error { + return Some(format!("the window track stopped early: {error}")); + } + if self.ticks >= 10 && self.late_ticks * 10 > self.ticks { + return Some(format!( + "the window track fell behind: {} of {} samples were late, so it is coarser than \ + the {} Hz asked for", + self.late_ticks, self.ticks, self.hz + )); + } + None + } +} + +/// Returns `hz` limited to [`MIN_HZ`]..=[`MAX_HZ`]. +#[must_use] +pub fn clamp_hz(hz: u32) -> u32 { + hz.clamp(MIN_HZ, MAX_HZ) +} + +/// The sampler thread of a recording that was asked for a window track. +pub struct WindowTracker { + stop: Arc, + handle: Option>, + hz: u32, +} + +impl WindowTracker { + /// Starts sampling into `windows.jsonl` in `dir`, at `hz` (limited by [`clamp_hz`]). + /// + /// # Errors + /// + /// The file cannot be created, or the thread cannot be spawned. + pub fn start(dir: &Path, hz: u32) -> eyre::Result { + let hz = clamp_hz(hz); + let path = dir.join(WINDOWS_FILE); + let file = + File::create(&path).wrap_err_with(|| format!("could not create {}", path.display()))?; + let stop = Arc::new(AtomicBool::new(false)); + let flag = Arc::clone(&stop); + let handle = std::thread::Builder::new() + .name("window-track".to_string()) + .spawn(move || sample_loop(BufWriter::new(file), hz, &flag)) + .wrap_err("could not start the window track thread")?; + Ok(Self { + stop, + handle: Some(handle), + hz, + }) + } + + /// The manifest entry for a track that has just started and has nothing yet. + #[must_use] + pub fn initial_info(&self) -> WindowTrackInfo { + WindowTrackInfo { + file: WINDOWS_FILE.to_string(), + hz: self.hz, + ..WindowTrackInfo::default() + } + } + + /// Stops the sampler, waits for it to flush and close the file, and returns what it did. + pub fn stop(mut self) -> WindowTrackInfo { + self.finish() + } + + fn finish(&mut self) -> WindowTrackInfo { + self.stop.store(true, Ordering::SeqCst); + let mut info = self.initial_info(); + match self.handle.take().map(JoinHandle::join) { + Some(Ok(done)) => info = done, + Some(Err(_)) => info.error = Some("the sampler thread panicked".to_string()), + None => {} + } + info + } +} + +impl Drop for WindowTracker { + fn drop(&mut self) { + if self.handle.is_some() { + let _ = self.finish(); + } + } +} + +fn sample_loop(mut out: BufWriter, hz: u32, stop: &AtomicBool) -> WindowTrackInfo { + let interval = Duration::from_secs_f64(1.0 / f64::from(hz)); + let mut info = WindowTrackInfo { + file: WINDOWS_FILE.to_string(), + hz, + ..WindowTrackInfo::default() + }; + let mut filter = ChangeFilter::default(); + let mut due = Instant::now(); + let mut flushed = Instant::now(); + + while !stop.load(Ordering::SeqCst) { + let mach_ns = clock::mono_now_ns(); + let windows = crate::platform::window::list(); + info.ticks += 1; + + if let Some(row) = filter.observe(mach_ns, windows) { + let written = serde_json::to_writer(&mut out, &row) + .map_err(std::io::Error::from) + .and_then(|()| out.write_all(b"\n")); + if let Err(error) = written { + info.error = Some(error.to_string()); + break; + } + info.rows += 1; + info.keyframes += u64::from(row.key); + } + if flushed.elapsed() >= Duration::from_secs(1) { + let _ = out.flush(); + flushed = Instant::now(); + } + + due += interval; + let now = Instant::now(); + if now > due + interval { + // not catching up: that would be a burst of samples of the same instant + info.late_ticks += 1; + due = now; + } else if let Some(wait) = due.checked_duration_since(now) { + // in slices, so a stop is noticed within a tenth of a second at any rate + let mut left = wait; + while !left.is_zero() && !stop.load(Ordering::SeqCst) { + let slice = left.min(Duration::from_millis(100)); + std::thread::sleep(slice); + left = left.saturating_sub(slice); + } + } + } + + if let Err(error) = out.flush() { + info.error.get_or_insert_with(|| error.to_string()); + } + info +} + +#[cfg(test)] +mod tests { + use super::*; + + fn window(id: u32, left: i32) -> WindowSample { + WindowSample { + id, + owner: "app".to_string(), + title: Some("t".to_string()), + rect: [left, 0, left + 10, 10], + visible: true, + minimized: false, + cloaked: None, + } + } + + #[test] + fn only_changes_and_a_keyframe_each_second_are_written() { + let mut filter = ChangeFilter::default(); + let ms = 1_000_000; + let first = filter.observe(0, vec![window(1, 0)]).expect("first sample"); + assert!(!first.key); + // a still desktop, sampled at 30 Hz for 0.9 s, writes nothing + for tick in 1..=27 { + assert!(filter.observe(tick * 33 * ms, vec![window(1, 0)]).is_none()); + } + // a different title is not a change + let mut retitled = window(1, 0); + retitled.title = Some("other".to_string()); + assert!(filter.observe(900 * ms, vec![retitled]).is_none()); + // the keyframe falls due at one second + let key = filter + .observe(1000 * ms, vec![window(1, 0)]) + .expect("keyframe"); + assert!(key.key); + assert!(filter.observe(1033 * ms, vec![window(1, 0)]).is_none()); + } + + #[test] + fn a_move_a_new_window_a_state_change_and_a_reorder_each_count() { + let mut filter = ChangeFilter::default(); + filter.observe(0, vec![window(1, 0), window(2, 0)]); + let changed = |filter: &mut ChangeFilter, at, windows| filter.observe(at, windows); + assert!(changed(&mut filter, 1, vec![window(1, 5), window(2, 0)]).is_some()); + assert!( + changed( + &mut filter, + 2, + vec![window(1, 5), window(2, 0), window(3, 0)] + ) + .is_some() + ); + assert!( + changed( + &mut filter, + 3, + vec![window(2, 0), window(1, 5), window(3, 0)] + ) + .is_some() + ); + let mut hidden = window(3, 0); + hidden.cloaked = Some(true); + assert!(changed(&mut filter, 4, vec![window(2, 0), window(1, 5), hidden]).is_some()); + assert!(changed(&mut filter, 5, vec![window(2, 0)]).is_some()); + assert!(changed(&mut filter, 6, vec![window(2, 0)]).is_none()); + } + + #[test] + fn rows_round_trip_and_leave_out_what_is_empty() { + let row = TrackRow { + mach_ns: 5, + key: false, + windows: vec![window(1, 0)], + }; + let text = serde_json::to_string(&row).unwrap(); + assert!(!text.contains("key") && !text.contains("cloaked"), "{text}"); + assert_eq!(serde_json::from_str::(&text).unwrap(), row); + } + + #[test] + fn rates_are_clamped_and_a_slow_sampler_is_called_out() { + assert_eq!((clamp_hz(0), clamp_hz(30), clamp_hz(1000)), (1, 30, 120)); + let mut info = WindowTrackInfo { + ticks: 100, + late_ticks: 5, + hz: 30, + ..WindowTrackInfo::default() + }; + assert!(info.warning().is_none()); + info.late_ticks = 50; + assert!(info.warning().unwrap().contains("fell behind")); + info.error = Some("disk full".to_string()); + assert!(info.warning().unwrap().contains("disk full")); + } + + #[test] + fn the_sampler_writes_a_first_row_and_a_keyframe() { + let dir = std::env::temp_dir().join(format!("window-track-test-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let tracker = WindowTracker::start(&dir, 60).unwrap(); + std::thread::sleep(Duration::from_millis(1300)); + let info = tracker.stop(); + assert!(info.ticks > 10, "{info:?}"); + assert!(info.rows >= 1 && info.error.is_none(), "{info:?}"); + + let text = std::fs::read_to_string(dir.join(WINDOWS_FILE)).unwrap(); + let rows: Vec = text + .lines() + .map(|l| serde_json::from_str(l).unwrap()) + .collect(); + assert_eq!(rows.len() as u64, info.rows); + assert!( + rows.windows(2) + .all(|pair| pair[0].mach_ns < pair[1].mach_ns) + ); + let _ = std::fs::remove_dir_all(&dir); + } +} diff --git a/komorebi-recorder/src/testkit.rs b/komorebi-recorder/src/testkit.rs index ad9c9199..f1775138 100644 --- a/komorebi-recorder/src/testkit.rs +++ b/komorebi-recorder/src/testkit.rs @@ -35,6 +35,10 @@ pub fn wall_us(video_s: f64) -> i64 { WALL_AT_MACH_ZERO_US + (mach_ns(video_s) / 1_000) as i64 } +/// A window of a [`SessionBuilder::window_track`] row: its id, owner and +/// `[left, top, right, bottom]`. +pub type TrackedWindow = (u32, &'static str, [i32; 4]); + struct PendingVideo { display: u32, start_s: f64, @@ -72,6 +76,7 @@ pub struct SessionBuilder { komorebi_commit: Option, subscriber: Option, videos: Vec, + window_track: Vec, } impl SessionBuilder { @@ -95,9 +100,39 @@ impl SessionBuilder { komorebi_commit: None, subscriber: None, videos: Vec::new(), + window_track: Vec::new(), } } + /// A window track (`windows.jsonl`): at each `video_s`, these windows, as `(id, owner, rect)` + /// with `rect` as `[left, top, right, bottom]`, front to back. Every window is visible, not + /// minimized and not cloaked. Call it once; a second call adds more rows. + #[must_use] + pub fn window_track(mut self, rows: &[(f64, &[TrackedWindow])]) -> Self { + for (video_s, windows) in rows { + self.window_track + .push(crate::sources::windowtrack::TrackRow { + mach_ns: mach_ns(*video_s), + key: false, + windows: windows + .iter() + .map( + |(id, owner, rect)| crate::sources::windowtrack::WindowSample { + id: *id, + owner: (*owner).to_string(), + title: None, + rect: *rect, + visible: true, + minimized: false, + cloaked: Some(false), + }, + ) + .collect(), + }); + } + self + } + /// A display's video file starting `start_s` into the recording: `frames` frames at `fps`, /// each numbered (see [`write_numbered_video`]). Needs `ffmpeg`; without it no file is written /// and the session has no video, so tests that need one check [`ffmpeg_available`] first. @@ -293,6 +328,30 @@ impl SessionBuilder { } std::fs::write(session.path("events.jsonl"), rows).expect("events"); + if !self.window_track.is_empty() { + self.window_track.sort_by_key(|row| row.mach_ns); + let text: String = self + .window_track + .iter() + .map(|row| { + serde_json::to_string(row).expect("row") + + " +" + }) + .collect(); + std::fs::write( + session.path(crate::sources::windowtrack::WINDOWS_FILE), + text, + ) + .expect("window track"); + manifest.window_track = Some(crate::sources::windowtrack::WindowTrackInfo { + file: crate::sources::windowtrack::WINDOWS_FILE.to_string(), + hz: 30, + rows: self.window_track.len() as u64, + ..Default::default() + }); + } + self.log.sort_by(|a, b| a.0.total_cmp(&b.0)); // tracing_subscriber right-aligns the level in five columns let log: String = self diff --git a/komorebi-recorder/tests/video.rs b/komorebi-recorder/tests/video.rs index 7685a543..08b9e989 100644 --- a/komorebi-recorder/tests/video.rs +++ b/komorebi-recorder/tests/video.rs @@ -464,3 +464,94 @@ fn compare_sets_two_recordings_side_by_side() { assert!(text.status.success()); assert!(String::from_utf8_lossy(&text.stdout).contains("worst frame gap")); } + +/// The RGB of one pixel of a PNG. +fn pixel(png_path: &str, x: usize, y: usize) -> [u8; 3] { + let decoder = png::Decoder::new(std::io::BufReader::new( + std::fs::File::open(png_path).expect("png"), + )); + let mut reader = decoder.read_info().expect("info"); + let mut pixels = vec![0; reader.output_buffer_size().expect("size")]; + let info = reader.next_frame(&mut pixels).expect("frame"); + let at = (y * info.width as usize + x) * 4; + [pixels[at], pixels[at + 1], pixels[at + 2]] +} + +#[test] +fn a_sheet_draws_the_windows_of_the_moment_and_says_so_when_there_is_no_track() { + if !testkit::ffmpeg_available() { + eprintln!("skipped: ffmpeg is not available to make a test video"); + return; + } + + // the display is 1000 by 800 points: one window fills its left half, and from 2 s another + // is in front of the right + let (session, _) = SessionBuilder::new("video-sheet-windows") + .video(1, 0.0, 90, 30) + .window_track(&[ + (0.0, &[(7, "term", [0, 0, 500, 800])]), + ( + 2.0, + &[ + (9, "browser", [500, 0, 1000, 800]), + (7, "term", [0, 0, 500, 800]), + ], + ), + ]) + .build(); + let dir = session.dir.display().to_string(); + let plain_out = session.path("plain.png").display().to_string(); + let drawn_out = session.path("drawn.png").display().to_string(); + + let run = |windows: bool, out: &str| { + let mut full: Vec = [ + "sheet", &dir, "--at", "1", "--at", "2.5", "--width", "200", "--json", "--out", out, + ] + .iter() + .map(ToString::to_string) + .collect(); + if windows { + full.push("--windows".to_string()); + } + let refs: Vec<&str> = full.iter().map(String::as_str).collect(); + json(&refs) + }; + + let plain = run(false, &plain_out); + assert!(plain["data"]["frames"][0].get("windows").is_none()); + let drawn = run(true, &drawn_out); + let frames = drawn["data"]["frames"].as_array().expect("frames"); + assert_eq!(frames[0]["windows"], json!([7])); + // front to back + assert_eq!(frames[1]["windows"], json!([9, 7])); + assert_eq!(drawn["warnings"], plain["warnings"]); + + // the border of the left window runs down the middle of the first cell, and is not in a sheet + // without windows + let (x, y, width, height) = ( + frames[0]["x"].as_u64().expect("x") as usize, + frames[0]["y"].as_u64().expect("y") as usize, + frames[0]["width"].as_u64().expect("width") as usize, + frames[0]["height"].as_u64().expect("height") as usize, + ); + let edge = (x + width / 2 - 1, y + height / 2); + assert_ne!( + pixel(&drawn_out, edge.0, edge.1), + pixel(&plain_out, edge.0, edge.1) + ); + let inside = (x + width / 4, y + height / 2); + assert_eq!( + pixel(&drawn_out, inside.0, inside.1), + pixel(&plain_out, inside.0, inside.1) + ); + + // a session recorded without a track draws nothing, and says why + let (bare, _) = SessionBuilder::new("video-sheet-no-windows") + .video(1, 0.0, 90, 30) + .build(); + let bare_dir = bare.dir.display().to_string(); + let answer = json(&["sheet", &bare_dir, "--at", "1", "--windows", "--json"]); + let warnings = answer["warnings"].to_string(); + assert!(warnings.contains("no window track"), "{warnings}"); + assert!(answer["data"]["frames"][0].get("windows").is_none()); +}