From 52a45aa17b0ccb626868680456bd884f600a79e0 Mon Sep 17 00:00:00 2001 From: LGUG2Z Date: Tue, 27 Oct 2026 06:15:22 -0700 Subject: [PATCH] feat(winprobe): ask Windows what it thinks of a window komorebi for Windows hides a window by cloaking it in place, so its state cannot show whether a window is really hidden. `record --probe-windows` follows each window event with a read of the window's real state from Windows and, now and then, asks komorebi whether it manages the window. The analysis compares the two: a window komorebi shows that Windows hides, one it hides that Windows shows, and a visible window it does not manage. macOS refuses the flag with a message. --- docs/usage/recording-a-bug-repro.md | 13 + komorebi-recorder/src/analysis/blocks.rs | 2 + komorebi-recorder/src/analysis/classify.rs | 2 + komorebi-recorder/src/analysis/events.rs | 6 + komorebi-recorder/src/analysis/invariants.rs | 202 ++++++++ komorebi-recorder/src/analysis/mod.rs | 1 + komorebi-recorder/src/analysis/overview.rs | 1 + komorebi-recorder/src/analysis/probed.rs | 401 +++++++++++++++ komorebi-recorder/src/analysis/trace.rs | 38 ++ komorebi-recorder/src/cli/record.rs | 9 + komorebi-recorder/src/platform/win/window.rs | 69 +++ komorebi-recorder/src/platform/window.rs | 32 ++ komorebi-recorder/src/recorder.rs | 15 +- komorebi-recorder/src/report.rs | 34 ++ .../src/sources/komorebi_events.rs | 101 ++++ komorebi-recorder/src/sources/mod.rs | 2 + komorebi-recorder/src/sources/winprobe.rs | 481 ++++++++++++++++++ komorebi-recorder/src/testkit.rs | 13 + komorebi-recorder/tests/windows_flavor.rs | 110 ++++ 19 files changed, 1531 insertions(+), 1 deletion(-) create mode 100644 komorebi-recorder/src/analysis/probed.rs create mode 100644 komorebi-recorder/src/sources/winprobe.rs diff --git a/docs/usage/recording-a-bug-repro.md b/docs/usage/recording-a-bug-repro.md index 7387ec30..e9b41a3f 100644 --- a/docs/usage/recording-a-bug-repro.md +++ b/docs/usage/recording-a-bug-repro.md @@ -91,6 +91,7 @@ Useful options: | `--oslog-all` | One extra unfiltered system-wide stream (very large) | | `--no-oslog`, `--no-komorebi` | Skip the OS trace, or skip touching komorebi entirely | | `--no-video` | Do not capture the screen; record everything else on the same clock (see below) | +| `--probe-windows` | Windows: after each window event, record what Windows says about that window (see below) | | `--fps N`, `--retina`, `--codec hevc\|h264` | Video settings (30 fps cap, logical points, HEVC by default) | ### Without video @@ -101,6 +102,18 @@ display (or no Screen Recording permission), or when what is on the screen must nothing about displays. A session made this way has no video, so video time `00:00.000` is the moment the recording started, and there is nothing for `frames` to show. +### Probing windows on Windows + +komorebi for Windows hides a window by _cloaking_ it where it is, so its state cannot show whether a +window is really hidden. `record --probe-windows` follows each window event with a read of the +window's real state from Windows (visible, cloaked, minimized, where it is) and, now and then, asks +komorebi whether it manages the window and which rules matched. `overview` then compares the two: +a window komorebi shows that Windows has hidden, one it hides that Windows shows, and a visible +window it does not manage (usually correct, with komorebi's explanation). It costs a few system +calls per event and a query to komorebi the first time a window is seen, and it reads nothing but +those facts. It does nothing on a Mac, where a hidden window is parked in a corner and the state +says so. + ### Checking your setup ``` diff --git a/komorebi-recorder/src/analysis/blocks.rs b/komorebi-recorder/src/analysis/blocks.rs index 20420c55..4eeeaea4 100644 --- a/komorebi-recorder/src/analysis/blocks.rs +++ b/komorebi-recorder/src/analysis/blocks.rs @@ -421,6 +421,7 @@ fn push_for<'a>( /// state_id: None, /// info: Flavor::Macos.rules().describe_event(&moved), /// parse_error: None, +/// probe: None, /// }]; /// /// // the log names the subscriber the push went to, so the pairing is exact @@ -549,6 +550,7 @@ mod tests { state_id: None, info: Flavor::Macos.rules().describe_event(&value), parse_error: None, + probe: None, } } diff --git a/komorebi-recorder/src/analysis/classify.rs b/komorebi-recorder/src/analysis/classify.rs index a99ed81a..1ab63452 100644 --- a/komorebi-recorder/src/analysis/classify.rs +++ b/komorebi-recorder/src/analysis/classify.rs @@ -148,6 +148,7 @@ fn class_of(event: &EventRecord, rules: &dyn Rules) -> Classification { /// state_id: Some(state_id), /// info: rules.describe_event(&value), /// parse_error: None, +/// probe: None, /// }; /// /// let moved = json!({ @@ -245,6 +246,7 @@ mod tests { state_id, info: Flavor::Macos.rules().describe_event(&value), parse_error: None, + probe: None, } } diff --git a/komorebi-recorder/src/analysis/events.rs b/komorebi-recorder/src/analysis/events.rs index 744c5986..732d1953 100644 --- a/komorebi-recorder/src/analysis/events.rs +++ b/komorebi-recorder/src/analysis/events.rs @@ -22,6 +22,7 @@ use super::flavor::Rules; use crate::session::Session; use crate::sources::komorebi_events::EventRow; +use crate::sources::winprobe::WindowFacts; use color_eyre::eyre; use color_eyre::eyre::WrapErr; use schemars::JsonSchema; @@ -177,6 +178,10 @@ pub struct EventRecord { /// Why the notification's JSON could not be parsed, if it could not. Such a record has /// [`Family::Unknown`] and the name `unreadable`. pub parse_error: Option, + /// The facts Windows reported about the event's window just after it, when the recording was + /// made with `--probe-windows`. + #[serde(skip_serializing_if = "Option::is_none")] + pub probe: Option, } impl EventRecord { @@ -234,6 +239,7 @@ impl EventRecord { state_id: row.state_id, info, parse_error: row.parse_error.clone(), + probe: row.probe.clone(), } } } diff --git a/komorebi-recorder/src/analysis/invariants.rs b/komorebi-recorder/src/analysis/invariants.rs index 8c33ac0e..e2a86c55 100644 --- a/komorebi-recorder/src/analysis/invariants.rs +++ b/komorebi-recorder/src/analysis/invariants.rs @@ -46,6 +46,7 @@ use super::state::Place; use super::state::Rect; use super::state::Snapshot; use super::state::Window; +use crate::sources::winprobe::WindowFacts; use schemars::JsonSchema; use serde::Serialize; use std::collections::BTreeMap; @@ -132,6 +133,207 @@ fn name(window: &Window) -> String { format!("{} \"{}\" (window {})", window.exe, window.title, window.id) } +/// Whether komorebi's state says a window should be on screen: `Some(true)` for the window in front +/// on a workspace that is showing, `Some(false)` for one that should be hidden (another workspace, +/// a tab behind the front one, something else filling the workspace), `None` where the state does +/// not say (a floating window). +fn should_be_shown(snapshot: &Snapshot, location: &Location) -> Option { + if !location.workspace_visible { + return Some(false); + } + + let workspace = snapshot + .monitors + .iter() + .find(|monitor| monitor.id == location.monitor_id)? + .workspaces + .iter() + .find(|workspace| workspace.name == location.workspace)?; + + match location.place { + Place::Container { index, tab } => { + if workspace.monocle.is_some() || workspace.maximized.is_some() { + // one window fills the workspace and the tiled ones are put away + return Some(false); + } + Some(workspace.containers.get(index)?.focused == tab) + } + Place::Monocle { tab } => Some(workspace.monocle.as_ref()?.focused == tab), + Place::Maximized => Some(true), + Place::Floating { .. } => None, + } +} + +/// Checks what Windows reported about one window against komorebi's state after the event about it. +/// +/// Only komorebi for Windows has this to check: it hides a window by cloaking it where it is, so +/// `State` cannot show whether a window really is hidden. `facts` is what the probe read just +/// after the event (see `WindowFacts` in `crate::sources::winprobe`), `snapshot` the state +/// komorebi sent with it, and `exe` the program the event named, used only to name a window +/// komorebi does not have. The result is empty if: +/// +/// - the window no longer exists (`facts.exists` is `false`), since a destroyed window is nobody's +/// business; +/// - komorebi is paused, since it is not managing windows; +/// - `snapshot` is from komorebi for Mac, whose hidden windows are found by [`check`]. +/// +/// Otherwise the window, found in the state by its handle, is judged: +/// +/// - `visible-window-hidden` (error): komorebi has the window in front on a workspace that is +/// showing, and Windows has it cloaked, minimized or not visible; +/// - `hidden-window-visible` (warn): komorebi has it put away (on a workspace that is not +/// showing, behind the front tab of its container, or behind a monocle or maximized window), and +/// Windows has it shown on a monitor; +/// - `unmanaged-window` (info): komorebi does not have it at all, and Windows has it shown. It is +/// not a fault in itself, since most dialogs and tooltips are rightly left alone, so it is +/// information, with komorebi's own explanation when the probe asked for one. Only the first +/// non-blank line of that explanation is quoted. +/// +/// A window in a floating place is not judged either way, since the state does not say whether a +/// floating window should be showing. +/// +/// A window komorebi is mid-way through moving can be caught in between: like every violation this +/// is evidence, and one that does not last is reported as the transition it probably was (see +/// `overview`). The function does not fail or panic. +/// +/// # Example +/// +/// ```rust +/// use komorebi_recorder::analysis::flavor::Flavor; +/// use komorebi_recorder::analysis::invariants::Severity; +/// use komorebi_recorder::analysis::invariants::check_probed; +/// use komorebi_recorder::analysis::state::Snapshot; +/// use komorebi_recorder::sources::winprobe::WindowFacts; +/// use komorebi_recorder::testkit::windows_state; +/// use komorebi_recorder::testkit::windows_window; +/// +/// // komorebi has Chrome in front on the workspace that is showing ... +/// let state = windows_state( +/// 65_537, +/// vec![vec![windows_window(131_072, "chrome.exe", Some([0, 0, 960, 1040]))]], +/// ); +/// let snapshot = Snapshot::parse(Flavor::Windows, &state); +/// +/// // ... and Windows says it is cloaked +/// let facts = WindowFacts { +/// hwnd: 131_072, +/// at_ns: 0, +/// exists: true, +/// visible: true, +/// cloaked: true, +/// minimized: false, +/// rect: Some([0, 0, 960, 1040]), +/// komorebi_says: None, +/// }; +/// let found = check_probed(&snapshot, &facts, Some("chrome.exe")); +/// assert_eq!(found.len(), 1); +/// assert_eq!(found[0].kind, "visible-window-hidden"); +/// assert_eq!(found[0].severity, Severity::Error); +/// assert!(found[0].message.contains("cloaked")); +/// +/// // a visible window komorebi does not have is information +/// let stray = WindowFacts { hwnd: 999, cloaked: false, ..facts.clone() }; +/// let found = check_probed(&snapshot, &stray, Some("dialog.exe")); +/// assert_eq!(found[0].kind, "unmanaged-window"); +/// assert_eq!(found[0].severity, Severity::Info); +/// +/// // a state from komorebi for Mac is not judged by what Windows says +/// let mac = Snapshot::parse(Flavor::Macos, &state); +/// assert!(check_probed(&mac, &facts, None).is_empty()); +/// ``` +/// +/// Covered by the unit tests of `analysis::probed` and by +/// `tests/windows_flavor.rs::a_window_komorebi_shows_that_windows_cloaked_is_a_finding_in_the_overview`. +#[must_use] +pub fn check_probed(snapshot: &Snapshot, facts: &WindowFacts, exe: Option<&str>) -> Vec { + if !facts.exists || snapshot.paused || snapshot.hides_windows_by_position() { + return Vec::new(); + } + + let id = u64::from(facts.hwnd); + let named = |window: Option<&Window>| match (window, exe) { + (Some(window), _) => name(window), + (None, Some(exe)) => format!("{exe} window {id}"), + (None, None) => format!("window {id}"), + }; + + let found = snapshot + .windows() + .into_iter() + .find(|(_, window)| window.id == id); + let Some((location, window)) = found else { + if !facts.shown() { + return Vec::new(); + } + + let why = facts + .komorebi_says + .as_deref() + .and_then(|says| says.lines().map(str::trim).find(|line| !line.is_empty())) + .map_or_else(String::new, |line| format!("; komorebi says: {line}")); + return vec![violation( + "unmanaged-window", + Severity::Info, + format!( + "{} is on screen but komorebi does not manage it{why}", + named(None) + ), + vec![id], + None, + )]; + }; + + match should_be_shown(snapshot, &location) { + Some(true) => facts + .hidden_how() + .map(|how| { + violation( + "visible-window-hidden", + Severity::Error, + format!( + "{} is in front on {}, which is showing, but Windows has it {how}", + named(Some(window)), + location.describe() + ), + vec![id], + Some(&location), + ) + }) + .into_iter() + .collect(), + Some(false) if facts.shown() => { + let on_screen = facts.rect.is_none_or(|[left, top, right, bottom]| { + let rect = Rect { + left: i64::from(left), + top: i64::from(top), + width: i64::from(right - left), + height: i64::from(bottom - top), + }; + snapshot + .monitors + .iter() + .any(|monitor| monitor.size.overlap_area(&rect) > 0) + }); + if !on_screen { + return Vec::new(); + } + + vec![violation( + "hidden-window-visible", + Severity::Warn, + format!( + "{} is on {}, which is not showing its windows, but Windows has it on screen", + named(Some(window)), + location.describe() + ), + vec![id], + Some(&location), + )] + } + _ => Vec::new(), + } +} + /// Checks one state and returns every violation found, most serious first. /// /// The result is sorted by severity and then by kind, so errors come before warnings, and is empty diff --git a/komorebi-recorder/src/analysis/mod.rs b/komorebi-recorder/src/analysis/mod.rs index da95f5d6..f7977818 100644 --- a/komorebi-recorder/src/analysis/mod.rs +++ b/komorebi-recorder/src/analysis/mod.rs @@ -28,6 +28,7 @@ pub mod logparse; pub mod lookup; pub mod os_trace; pub mod overview; +pub mod probed; pub mod schemas; pub mod search; pub mod sheet; diff --git a/komorebi-recorder/src/analysis/overview.rs b/komorebi-recorder/src/analysis/overview.rs index 398ec45c..9a1f9987 100644 --- a/komorebi-recorder/src/analysis/overview.rs +++ b/komorebi-recorder/src/analysis/overview.rs @@ -408,6 +408,7 @@ impl Analysis { pub fn findings(&self) -> eyre::Result> { let mut found = Vec::new(); self.find_state_problems(&mut found)?; + self.find_probed_problems(&mut found)?; self.find_event_problems(&mut found); self.find_log_problems(&mut found); self.find_incidents(&mut found); diff --git a/komorebi-recorder/src/analysis/probed.rs b/komorebi-recorder/src/analysis/probed.rs new file mode 100644 index 00000000..f62ecf69 --- /dev/null +++ b/komorebi-recorder/src/analysis/probed.rs @@ -0,0 +1,401 @@ +//! Findings from what Windows said about windows, as the recording's `--probe-windows` stored it. +//! +//! Each probed event gives [`invariants::check_probed`] a window's real state to compare with +//! komorebi's. One problem usually shows on several events in a row (a window komorebi has in front +//! and Windows has cloaked stays so until something changes it), so the same problem on the same +//! window across consecutive probes is one finding that says how long it lasted, as for the other +//! checks, not one per event. +//! +//! This module has no public items. It adds to the detectors of [`Analysis::findings`], which runs +//! it with the others, so the `overview` command and `brief` list its findings among theirs: it +//! is where komorebi for Windows' hidden-window violations come from (see +//! [`super::invariants`]), since a Windows state cannot say whether a window is hidden. +//! +//! # What is made of a probe +//! +//! Only an event that has both a probe (a `crate::sources::winprobe::WindowFacts`) and a state is +//! looked at, so a session recorded without `--probe-windows`, and any session of komorebi for +//! Mac, has no findings from here. Of those: +//! +//! - A probe read more than 1 s after its event is skipped: the window may have changed since. +//! - An event about one of the recorder's own windows (a program whose name starts with +//! `komorebi-recorder`) is skipped, because komorebi does not manage them by design. +//! - The rest are judged by [`invariants::check_probed`]. A problem is followed for each window +//! separately: it lasts from the first probe of that window that has it until the first later +//! probe of the same window that does not. A problem still there at the window's last probe is +//! marked [`Finding::persisted_to_end`]. +//! +//! Each such run is one [`Finding`] of kind `invariant:`, with the violation's severity and a +//! summary of how many probes saw it and over how long. A problem seen at one probe only, +//! and gone at the window's next, is taken for a transition in progress, and its severity is +//! lowered one step (error to warn, anything else to info). The finding's `refs` names the first +//! event it was seen on and its `next` is the `trace` command for it. +//! +//! At most 20 `unmanaged-window` findings are reported for a recording: komorebi leaves toasts, +//! tooltips and dialogs alone on purpose, and a long recording sees very many. +//! +//! # Example +//! +//! Chrome is in front on the workspace that is showing, and after each of two events Windows says +//! it is cloaked, so it is one finding that lasted a second and was still so at the end: +//! +//! ```rust +//! use komorebi_recorder::analysis::Analysis; +//! use komorebi_recorder::analysis::flavor::Flavor; +//! use komorebi_recorder::sources::winprobe::WindowFacts; +//! use komorebi_recorder::testkit::SessionBuilder; +//! use komorebi_recorder::testkit::windows_state; +//! use komorebi_recorder::testkit::windows_window; +//! use serde_json::json; +//! +//! const CHROME: i64 = 131_072; +//! +//! let chrome = || windows_window(CHROME, "chrome.exe", Some([0, 0, 960, 1040])); +//! let state = || windows_state(65_537, vec![vec![chrome()]]); +//! let focus = json!({"type": "FocusChange", "content": ["SystemForeground", chrome()]}); +//! let cloaked = WindowFacts { +//! hwnd: CHROME as u32, +//! at_ns: 0, +//! exists: true, +//! visible: true, +//! cloaked: true, +//! minimized: false, +//! rect: Some([0, 0, 960, 1040]), +//! komorebi_says: None, +//! }; +//! +//! let (session, _) = SessionBuilder::new("doc-probed") +//! .flavor(Flavor::Windows) +//! .event(2.0, focus.clone(), Some(state())) +//! .probe(cloaked.clone()) +//! .event(3.0, focus, Some(state())) +//! .probe(cloaked) +//! .build(); +//! +//! let findings = Analysis::open(&session.dir)?.findings()?; +//! assert_eq!(findings.len(), 1); +//! assert_eq!(findings[0].kind, "invariant:visible-window-hidden"); +//! assert!(findings[0].summary.contains("2 probe(s) over 1.0 s")); +//! assert!(findings[0].persisted_to_end); +//! # Ok::<(), Box>(()) +//! ``` +//! +//! The other cases are covered by this module's unit tests, among them +//! `a_problem_that_is_gone_at_the_next_probe_was_a_transition_and_is_reported_as_one`, +//! `a_window_behind_the_front_one_that_windows_has_on_screen_is_leaking`, +//! `a_visible_window_komorebi_does_not_have_is_information_with_its_explanation` and +//! `our_own_windows_are_unmanaged_on_purpose_and_a_destroyed_one_is_nobodys_business`, and by +//! `tests/windows_flavor.rs::a_window_komorebi_shows_that_windows_cloaked_is_a_finding_in_the_overview`. + +use super::Analysis; +use super::invariants; +use super::invariants::Severity; +use super::overview::Finding; +use std::collections::HashMap; + +/// A probe read this long after its event may be of a window that has since changed. +const STALE_NS: u64 = 1_000_000_000; + +/// A problem seen once, for less than this, is usually a transition in progress. +const TRANSIENT_NS: u64 = 500_000_000; + +/// The most windows komorebi does not manage that are reported: toasts, tooltips and dialogs are +/// rightly left alone, and a long recording sees very many. +const UNMANAGED_LIMIT: usize = 20; + +struct Run { + violation: invariants::Violation, + first_seq: u64, + first_ns: u64, + last_ns: u64, + probes: usize, +} + +impl Analysis { + pub(super) fn find_probed_problems( + &self, + found: &mut Vec, + ) -> color_eyre::eyre::Result<()> { + let dir = self.session.dir.display().to_string(); + let mut open: HashMap<(String, u64), Run> = HashMap::new(); + let mut done: Vec<(Run, Option)> = Vec::new(); + + for event in self.events() { + let (Some(facts), Some(state_id)) = (&event.probe, event.state_id) else { + continue; + }; + if facts.at_ns.saturating_sub(event.recv_mach_ns) > STALE_NS { + continue; + } + + // komorebi does not manage this program's own windows, by design + let exe = event.info.exe.as_deref(); + if exe.is_some_and(|exe| exe.to_ascii_lowercase().starts_with("komorebi-recorder")) { + continue; + } + + let snapshot = self.snapshot(state_id)?; + let current = invariants::check_probed(&snapshot, facts, exe); + + // what was wrong about this window and no longer is + let id = u64::from(facts.hwnd); + let cleared: Vec<(String, u64)> = open + .keys() + .filter(|(kind, window)| { + *window == id && !current.iter().any(|violation| &violation.kind == kind) + }) + .cloned() + .collect(); + for key in cleared { + if let Some(run) = open.remove(&key) { + done.push((run, Some(event.recv_mach_ns))); + } + } + + for violation in current { + let key = (violation.kind.clone(), id); + match open.get_mut(&key) { + Some(run) => { + run.last_ns = event.recv_mach_ns; + run.probes += 1; + } + None => { + open.insert( + key, + Run { + violation, + first_seq: event.seq, + first_ns: event.recv_mach_ns, + last_ns: event.recv_mach_ns, + probes: 1, + }, + ); + } + } + } + } + + done.extend(open.into_values().map(|run| (run, None))); + done.sort_by_key(|(run, _)| run.first_ns); + + let mut unmanaged = 0; + for (run, cleared_at) in done { + if run.violation.kind == "unmanaged-window" { + unmanaged += 1; + if unmanaged > UNMANAGED_LIMIT { + continue; + } + } + + let lasted = run.last_ns.saturating_sub(run.first_ns); + let severity = if run.probes == 1 && lasted < TRANSIENT_NS && cleared_at.is_some() { + match run.violation.severity { + Severity::Error => Severity::Warn, + _ => Severity::Info, + } + } else { + run.violation.severity + }; + + let mut finding = self.finding( + &format!("invariant:{}", run.violation.kind), + severity, + format!( + "{}; Windows said so on {} probe(s) over {:.1} s{}", + run.violation.message, + run.probes, + lasted as f64 / 1e9, + if cleared_at.is_none() { + ", and at the last one still" + } else { + "" + } + ), + run.first_ns, + cleared_at.unwrap_or(run.last_ns), + ); + finding.refs = vec![format!("event:{}", run.first_seq)]; + finding.window_ids = run.violation.window_ids.clone(); + finding.persisted_to_end = cleared_at.is_none(); + finding.next = Some(format!( + "komorebi-recorder trace {dir} event:{}", + run.first_seq + )); + found.push(finding); + } + + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::analysis::flavor::Flavor; + use crate::sources::winprobe::WindowFacts; + use crate::testkit::SessionBuilder; + use crate::testkit::windows_state; + use crate::testkit::windows_window; + use serde_json::json; + + const FRONT: i64 = 131_072; + const BEHIND: i64 = 262_144; + + fn facts(hwnd: i64, cloaked: bool, visible: bool) -> WindowFacts { + WindowFacts { + hwnd: u32::try_from(hwnd).expect("hwnd"), + at_ns: 0, + exists: true, + visible, + cloaked, + minimized: false, + rect: Some([0, 0, 960, 1040]), + komorebi_says: None, + } + } + + fn focus(hwnd: i64, exe: &str) -> serde_json::Value { + json!({"type": "FocusChange", "content": ["SystemForeground", + windows_window(hwnd, exe, Some([0, 0, 960, 1040]))]}) + } + + /// One workspace showing a container with `FRONT` in front of `BEHIND`. + fn state() -> serde_json::Value { + let mut state = windows_state( + 65_537, + vec![vec![ + windows_window(FRONT, "chrome.exe", Some([0, 0, 960, 1040])), + windows_window(BEHIND, "code.exe", Some([0, 0, 960, 1040])), + ]], + ); + // the second tab is behind the first + state["monitors"]["elements"][0]["workspaces"]["elements"][0]["containers"]["elements"] + [0]["windows"]["focused"] = json!(0); + state + } + + fn findings(name: &str, build: impl FnOnce(SessionBuilder) -> SessionBuilder) -> Vec { + let (session, _) = build(SessionBuilder::new(name).flavor(Flavor::Windows)).build(); + let analysis = Analysis::open(&session.dir).expect("open"); + let mut found = Vec::new(); + analysis.find_probed_problems(&mut found).expect("findings"); + found + } + + #[test] + fn a_window_komorebi_shows_that_windows_has_cloaked_is_an_error_that_says_how_long() { + let found = findings("probed-cloaked", |builder| { + builder + .event(2.0, focus(FRONT, "chrome.exe"), Some(state())) + .probe(facts(FRONT, true, true)) + .event(3.0, focus(FRONT, "chrome.exe"), Some(state())) + .probe(facts(FRONT, true, true)) + }); + + assert_eq!(found.len(), 1, "{found:#?}"); + let finding = &found[0]; + assert_eq!(finding.kind, "invariant:visible-window-hidden"); + assert_eq!(finding.severity, Severity::Error); + assert!(finding.summary.contains("cloaked"), "{}", finding.summary); + assert!( + finding.summary.contains("2 probe(s) over 1.0 s"), + "{}", + finding.summary + ); + assert!(finding.persisted_to_end); + assert_eq!(finding.window_ids, [FRONT as u64]); + } + + #[test] + fn a_problem_that_is_gone_at_the_next_probe_was_a_transition_and_is_reported_as_one() { + let found = findings("probed-transient", |builder| { + builder + .event(2.0, focus(FRONT, "chrome.exe"), Some(state())) + .probe(facts(FRONT, true, true)) + .event(2.2, focus(FRONT, "chrome.exe"), Some(state())) + .probe(facts(FRONT, false, true)) + }); + + assert_eq!(found.len(), 1, "{found:#?}"); + assert_eq!( + found[0].severity, + Severity::Warn, + "downgraded from an error" + ); + assert!(!found[0].persisted_to_end); + } + + #[test] + fn a_window_behind_the_front_one_that_windows_has_on_screen_is_leaking() { + let found = findings("probed-leak", |builder| { + builder + .event(2.0, focus(BEHIND, "code.exe"), Some(state())) + .probe(facts(BEHIND, false, true)) + }); + + assert_eq!(found.len(), 1, "{found:#?}"); + assert_eq!(found[0].kind, "invariant:hidden-window-visible"); + } + + #[test] + fn a_window_that_is_hidden_the_way_komorebi_hides_one_is_not_a_finding() { + let found = findings("probed-fine", |builder| { + builder + // the window behind is cloaked, as komorebi hides it + .event(2.0, focus(BEHIND, "code.exe"), Some(state())) + .probe(facts(BEHIND, true, true)) + // and the one in front is shown + .event(3.0, focus(FRONT, "chrome.exe"), Some(state())) + .probe(facts(FRONT, false, true)) + }); + + assert!(found.is_empty(), "{found:#?}"); + } + + #[test] + fn a_visible_window_komorebi_does_not_have_is_information_with_its_explanation() { + let mut shown = facts(999, false, true); + shown.komorebi_says = + Some("not managed\n matched ignore rule: exe == dialog.exe".to_string()); + let found = findings("probed-unmanaged", |builder| { + builder + .event(2.0, focus(999, "dialog.exe"), Some(state())) + .probe(shown) + }); + + assert_eq!(found.len(), 1, "{found:#?}"); + assert_eq!(found[0].kind, "invariant:unmanaged-window"); + assert_eq!(found[0].severity, Severity::Info); + assert!( + found[0].summary.contains("komorebi says: not managed"), + "{}", + found[0].summary + ); + } + + #[test] + fn our_own_windows_are_unmanaged_on_purpose_and_a_destroyed_one_is_nobodys_business() { + let mut gone = facts(999, false, false); + gone.exists = false; + let found = findings("probed-own", |builder| { + builder + .event(2.0, focus(777, "komorebi-recorder.exe"), Some(state())) + .probe(facts(777, false, true)) + .event(3.0, focus(999, "dialog.exe"), Some(state())) + .probe(gone) + }); + + assert!(found.is_empty(), "{found:#?}"); + } + + #[test] + fn a_session_without_probes_has_no_probed_findings_and_a_mac_one_is_never_judged_by_them() { + let (session, _) = SessionBuilder::new("probed-none") + .event(2.0, json!({"type": "Retile"}), None) + .build(); + let analysis = Analysis::open(&session.dir).expect("open"); + let mut found = Vec::new(); + analysis.find_probed_problems(&mut found).expect("findings"); + assert!(found.is_empty()); + } +} diff --git a/komorebi-recorder/src/analysis/trace.rs b/komorebi-recorder/src/analysis/trace.rs index 5d357b14..e8bd3ca9 100644 --- a/komorebi-recorder/src/analysis/trace.rs +++ b/komorebi-recorder/src/analysis/trace.rs @@ -165,6 +165,25 @@ pub struct TraceData { /// [`Analysis::trace`] always leaves it `None`, and the caller fills it in. #[serde(skip_serializing_if = "Option::is_none")] pub os_trace: Option, + /// Windows' account of the event's window just after it, checked against komorebi's state: + /// only for a recording made with `--probe-windows`. + #[serde(skip_serializing_if = "Option::is_none")] + pub window_probe: Option, +} + +/// A window as Windows reported it, and the problems that makes in komorebi's state. +/// +/// Only a session recorded with `record --probe-windows` has these, and only for events about a +/// window. See [`check_probed`](super::invariants::check_probed) for what is compared. +#[derive(Debug, Clone, PartialEq, Serialize, JsonSchema)] +pub struct ProbeView { + /// Windows' account of the window (a `WindowFacts` from `crate::sources::winprobe`). + pub facts: crate::sources::winprobe::WindowFacts, + /// How long after the event it was read, in milliseconds: the longer, the less it shows. `0` if + /// it was read before the event was received. + pub read_after_ms: f64, + /// The problems the facts show in komorebi's state, if any: empty when they agree. + pub violations: Vec, } fn spans_of(line: &Line) -> String { @@ -450,6 +469,25 @@ impl Analysis { violations_resolved: resolved, notes, os_trace: None, + window_probe: self.window_probe(index), + }) + } + + /// What Windows said about event `index`'s window, if it was asked, checked against the state + /// komorebi reported with the event. + fn window_probe(&self, index: usize) -> Option { + let event = self.events().get(index)?; + let facts = event.probe.clone()?; + let violations = event + .state_id + .and_then(|state_id| self.snapshot(state_id).ok()) + .map(|snapshot| invariants::check_probed(&snapshot, &facts, event.info.exe.as_deref())) + .unwrap_or_default(); + + Some(ProbeView { + read_after_ms: facts.at_ns.saturating_sub(event.recv_mach_ns) as f64 / 1e6, + facts, + violations, }) } diff --git a/komorebi-recorder/src/cli/record.rs b/komorebi-recorder/src/cli/record.rs index 6dc0aa3b..42ca33a7 100644 --- a/komorebi-recorder/src/cli/record.rs +++ b/komorebi-recorder/src/cli/record.rs @@ -62,6 +62,13 @@ pub struct RecordArgs { /// Do not capture komorebi's log or events (video and OS trace only) #[arg(long)] pub no_komorebi: bool, + /// komorebi for Windows hides a window by cloaking it in place, which its state cannot show. + /// With this, each window event is followed by a read of what Windows says about that window + /// (visible, cloaked, minimized, where it is) and, now and then, komorebi's own answer to "would + /// you manage this window, and why", so that a window komorebi shows but Windows has hidden (or + /// the reverse) can be found. Windows only + #[arg(long)] + pub probe_windows: bool, /// Do not capture the screen: record everything else (komorebi's events and log, the OS /// trace and incidents) on the same clock #[arg(long, conflicts_with = "displays")] @@ -106,6 +113,7 @@ impl Default for RecordArgs { codec: Codec::default(), no_komorebi: false, no_video: false, + probe_windows: false, no_oslog: false, oslog_apps: AppsMode::Managed, oslog_all: false, @@ -170,6 +178,7 @@ impl RecordArgs { duration: self.duration.map(Duration::from_secs_f64), no_komorebi: self.no_komorebi, no_video: self.no_video, + probe_windows: self.probe_windows, } } } diff --git a/komorebi-recorder/src/platform/win/window.rs b/komorebi-recorder/src/platform/win/window.rs index 32eafbc5..3f8ce4ed 100644 --- a/komorebi-recorder/src/platform/win/window.rs +++ b/komorebi-recorder/src/platform/win/window.rs @@ -10,17 +10,26 @@ //! |---|---| //! | [`exclude_from_capture`] | asking for this process's windows to be left out of captures | //! | [`own_window_handles`] | the handles of those windows, as komorebi knows windows | +//! | [`inspect`] | what Windows says about any window, for the per-event probe | //! //! This module is not part of the facade: [`crate::platform::window`], which is shared by every //! host, calls it. Nothing tests it automatically; it acts on real windows, and is checked on a //! real machine (see `docs/dev/windows-checkpoints.md`). +use crate::sources::winprobe::WindowFacts; use std::ffi::c_void; use windows::Win32::Foundation::HWND; use windows::Win32::Foundation::LPARAM; +use windows::Win32::Foundation::RECT; +use windows::Win32::Graphics::Dwm::DWMWA_CLOAKED; +use windows::Win32::Graphics::Dwm::DWMWA_EXTENDED_FRAME_BOUNDS; +use windows::Win32::Graphics::Dwm::DwmGetWindowAttribute; use windows::Win32::System::Threading::GetCurrentProcessId; use windows::Win32::UI::WindowsAndMessaging::EnumWindows; +use windows::Win32::UI::WindowsAndMessaging::GetWindowRect; use windows::Win32::UI::WindowsAndMessaging::GetWindowThreadProcessId; +use windows::Win32::UI::WindowsAndMessaging::IsIconic; +use windows::Win32::UI::WindowsAndMessaging::IsWindow; use windows::Win32::UI::WindowsAndMessaging::IsWindowVisible; use windows::Win32::UI::WindowsAndMessaging::SetWindowDisplayAffinity; use windows::Win32::UI::WindowsAndMessaging::WDA_EXCLUDEFROMCAPTURE; @@ -73,3 +82,63 @@ pub fn exclude_from_capture() -> usize { }) .count() } + +/// Returns what Windows says about the window with handle `hwnd` right now: whether it exists, is +/// visible, cloaked or minimized, and where it is. The rect is the visible frame +/// (`DWMWA_EXTENDED_FRAME_BOUNDS`, or `GetWindowRect` if that cannot be read), which is how +/// komorebi reads one, so the two can be compared. +/// +/// This is always `Some`: a handle that names no window is a [`WindowFacts`] with `exists` false +/// and nothing else read. `at_ns` and `komorebi_says` are left for the caller to fill in. It is +/// reached through [`crate::platform::window::inspect`]. +#[must_use] +pub fn inspect(hwnd: u32) -> Option { + let window = HWND(hwnd as usize as *mut c_void); + let mut facts = WindowFacts { + hwnd, + at_ns: 0, + exists: false, + visible: false, + cloaked: false, + minimized: false, + rect: None, + komorebi_says: None, + }; + + // SAFETY: queries about a handle that may be stale, which they report as a failure; every out + // pointer is valid for its call + unsafe { + if !IsWindow(Some(window)).as_bool() { + return Some(facts); + } + + facts.exists = true; + facts.visible = IsWindowVisible(window).as_bool(); + facts.minimized = IsIconic(window).as_bool(); + + // non-zero for a window cloaked by the shell (another virtual desktop), by its application, + // or by komorebi + let mut cloaked = 0_u32; + let _ = DwmGetWindowAttribute( + window, + DWMWA_CLOAKED, + (&raw mut cloaked).cast(), + size_of::() as u32, + ); + facts.cloaked = cloaked != 0; + + let mut frame = RECT::default(); + let framed = DwmGetWindowAttribute( + window, + DWMWA_EXTENDED_FRAME_BOUNDS, + (&raw mut frame).cast(), + size_of::() as u32, + ) + .is_ok(); + if framed || GetWindowRect(window, &raw mut frame).is_ok() { + facts.rect = Some([frame.left, frame.top, frame.right, frame.bottom]); + } + } + + Some(facts) +} diff --git a/komorebi-recorder/src/platform/window.rs b/komorebi-recorder/src/platform/window.rs index 38b4b9ec..b9598e87 100644 --- a/komorebi-recorder/src/platform/window.rs +++ b/komorebi-recorder/src/platform/window.rs @@ -12,8 +12,11 @@ //! again on Windows: windows are created after the last call (the viewer opened from the recorder, //! a dialog), so the request to be left out of captures is repeated. On macOS it does nothing. //! +//! [`inspect`] is a separate matter: it reads what Windows says about any window handle, for the +//! per-event probe (`crate::sources::winprobe`), and is `None` on every other host. use crate::komorebi_link; +use crate::sources::winprobe::WindowFacts; use std::time::Duration; use std::time::Instant; @@ -34,6 +37,35 @@ const PROBE_AFTER: Duration = Duration::from_secs(3); /// has no such windows the file says so. pub const PROBE_VARIABLE: &str = "KOMOREBI_RECORDER_PROBE_WINDOW"; +/// Returns what Windows says about the window with handle `hwnd` right now (see +/// `crate::sources::winprobe`), or `None` on a host that has no such windows to ask about. +/// +/// On Windows the answer is always `Some`, even for a handle that names no window: that is a +/// `WindowFacts` whose `exists` is `false`. The rect it carries is the visible frame, which is how +/// komorebi reads one, so the two can be compared. `at_ns` and `komorebi_says` are not filled in; +/// the prober that calls this sets them. +/// +/// # Example +/// +/// ```rust +/// use komorebi_recorder::platform::window::inspect; +/// +/// // only Windows has windows to ask about +/// assert_eq!(inspect(0x20000).is_some(), cfg!(windows)); +/// ``` +#[must_use] +pub fn inspect(hwnd: u32) -> Option { + #[cfg(windows)] + { + super::win::window::inspect(hwnd) + } + #[cfg(not(windows))] + { + let _ = hwnd; + None + } +} + /// Per-window housekeeping for a window's update loop. Call [`tick`](Chrome::tick) on every frame: /// it is cheap, and does something about once a second. /// diff --git a/komorebi-recorder/src/recorder.rs b/komorebi-recorder/src/recorder.rs index 2a630de7..61223e89 100644 --- a/komorebi-recorder/src/recorder.rs +++ b/komorebi-recorder/src/recorder.rs @@ -52,6 +52,7 @@ use crate::sources::oslog; use crate::sources::oslog::OsLogLevel; use crate::sources::oslog::StreamLive; use crate::sources::oslog::StreamSet; +use crate::sources::winprobe::EventProber; use crate::sources::wintrace::TraceSet; use crate::system_state; use crate::system_state::SessionTracker; @@ -166,6 +167,10 @@ pub struct RecordOptions { /// Do not capture the screen: everything else is recorded, on the same clock. For a headless /// machine, or when what is on the screen must not be recorded. pub no_video: bool, + /// After each komorebi window event, read what Windows says about that window (visible, + /// cloaked, minimized, where) and, now and then, ask komorebi whether it manages it. komorebi + /// for Windows hides windows by cloaking them in place, which `State` cannot show. + pub probe_windows: bool, } /// Where a recording is in its life. @@ -726,6 +731,13 @@ impl<'a> Recording<'a> { /// Start every source, in the order that makes the non-video sources a superset of the video. #[allow(clippy::too_many_lines)] fn begin(options: &'a RecordOptions, live: &'a Live) -> eyre::Result { + if options.probe_windows && !cfg!(windows) { + eyre::bail!( + "--probe-windows reads the state of komorebi for Windows' windows from Windows, \ + and this is not Windows" + ); + } + // without video there is no screen to ask about, or permission to ask for let present = if options.no_video { Vec::new() @@ -947,7 +959,8 @@ impl<'a> Recording<'a> { let events = if options.no_komorebi { None } else { - match EventRecorder::start(&session.dir, Some(discovery_tx)) { + let prober = options.probe_windows.then(EventProber::windows); + match EventRecorder::start_with(&session.dir, Some(discovery_tx), prober) { Ok(recorder) => { manifest.komorebi.subscriber = Some(recorder.subscriber_name().to_string()); Some(recorder) diff --git a/komorebi-recorder/src/report.rs b/komorebi-recorder/src/report.rs index dc02863d..be58b371 100644 --- a/komorebi-recorder/src/report.rs +++ b/komorebi-recorder/src/report.rs @@ -1065,6 +1065,40 @@ pub fn trace_text(data: &TraceData) -> String { let _ = writeln!(out, "\nwrong before this event, and not after:"); violation_lines(&mut out, &data.violations_resolved); } + if let Some(probe) = &data.window_probe { + let facts = &probe.facts; + let _ = writeln!( + out, + "\nwhat Windows said about window {} ({:.0} ms after the event):", + facts.hwnd, probe.read_after_ms + ); + let _ = writeln!( + out, + " exists {}, visible {}, cloaked {}, minimized {}{}", + facts.exists, + facts.visible, + facts.cloaked, + facts.minimized, + facts + .rect + .map_or_else(String::new, |[left, top, right, bottom]| format!( + ", at {}x{} ({left}, {top})", + right - left, + bottom - top + )) + ); + if let Some(says) = &facts.komorebi_says { + let _ = writeln!(out, " komorebi says:"); + for line in says.lines() { + let _ = writeln!(out, " {line}"); + } + } + if probe.violations.is_empty() { + let _ = writeln!(out, " which agrees with komorebi's state"); + } else { + violation_lines(&mut out, &probe.violations); + } + } if let Some(os) = &data.os_trace { os_trace_text(&mut out, os); } diff --git a/komorebi-recorder/src/sources/komorebi_events.rs b/komorebi-recorder/src/sources/komorebi_events.rs index 954dabc3..c793226a 100644 --- a/komorebi-recorder/src/sources/komorebi_events.rs +++ b/komorebi-recorder/src/sources/komorebi_events.rs @@ -52,6 +52,8 @@ use crate::komorebi_link::Message; use crate::komorebi_link::SubscribeOptions; use crate::komorebi_link::UnixListener; use crate::komorebi_link::UnixStream; +use crate::sources::winprobe::EventProber; +use crate::sources::winprobe::WindowFacts; use color_eyre::eyre; use color_eyre::eyre::WrapErr; use crossbeam_channel::Receiver; @@ -153,6 +155,11 @@ pub struct EventRow { /// The payload as text, with any invalid UTF-8 replaced. Present only in an unreadable row. #[serde(default, skip_serializing_if = "Option::is_none")] pub raw: Option, + /// What Windows said about the window the event is about, when recording with + /// `--probe-windows` (see `crate::sources::winprobe`). Absent for events that are not about a + /// window, and on every row of a recording made without the probe. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub probe: Option, } /// Something the app-discovery logic wants to know about as it happens, sent by the writer thread. @@ -288,6 +295,21 @@ impl EventRecorder { /// /// An error from the subscription itself is not one of them. pub fn start(dir: &Path, discovery: Option>) -> eyre::Result { + Self::start_with(dir, discovery, None) + } + + /// [`EventRecorder::start`], with a prober that reads the real state of the window each event is + /// about (`--probe-windows`). It runs on the writer thread, which is also where the events are + /// parsed: the accept thread does nothing but receive. + /// + /// # Errors + /// + /// The same as [`EventRecorder::start`]. + pub fn start_with( + dir: &Path, + discovery: Option>, + prober: Option, + ) -> eyre::Result { std::fs::create_dir_all(dir.join("states"))?; let name = format!("komorebi-recorder-{}", std::process::id()); @@ -311,12 +333,14 @@ impl EventRecorder { let writer = std::thread::Builder::new() .name("komorebi-events-write".to_string()) .spawn(move || { + let mut prober = prober; write_loop( &rx, events_file, &states_dir, &writer_counters, discovery.as_ref(), + prober.as_mut(), ) })?; @@ -470,6 +494,7 @@ fn write_loop( states_dir: &Path, counters: &EventCounters, discovery: Option<&Sender>, + mut prober: Option<&mut EventProber>, ) { let mut events = BufWriter::new(events_file); let mut previous_state: Option = None; @@ -508,6 +533,11 @@ fn write_loop( previous_state = Some(state); } + let probe = prober + .as_mut() + .zip(event.as_ref()) + .and_then(|(prober, event)| prober.probe(event, clock::mono_now_ns())); + EventRow { seq: raw.seq, recv_wall_us: raw.recv_wall_us, @@ -516,6 +546,7 @@ fn write_loop( state_id: (state_id > 0).then_some(state_id), parse_error: None, raw: None, + probe, } } Err(error) => { @@ -528,6 +559,7 @@ fn write_loop( state_id: (state_id > 0).then_some(state_id), parse_error: Some(error.to_string()), raw: Some(String::from_utf8_lossy(&raw.bytes).into_owned()), + probe: None, } } }; @@ -601,9 +633,11 @@ mod tests { state_id: Some(2), parse_error: None, raw: None, + probe: None, }; let text = serde_json::to_string(&row).expect("serialize"); assert!(!text.contains("parse_error")); + assert!(!text.contains("probe")); let back: EventRow = serde_json::from_str(&text).expect("parse"); assert_eq!(back.seq, 3); assert_eq!(back.state_id, Some(2)); @@ -644,6 +678,7 @@ mod tests { &dir.join("states"), &counters, None, + None, ); assert_eq!(counters.events.load(Ordering::Relaxed), 4); @@ -670,4 +705,70 @@ mod tests { assert!(dir.join("states/000002.json").exists()); assert!(!dir.join("states/000003.json").exists()); } + + #[test] + fn the_writer_stores_what_the_prober_read_beside_the_event_it_is_about() { + let dir = std::env::temp_dir().join(format!( + "komorebi-recorder-events-probe-{}", + std::process::id() + )); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(dir.join("states")).expect("dir"); + + let window = serde_json::json!({"hwnd": 131072, "title": "t", "exe": "a.exe", "class": "C", + "rect": {"left": 0, "top": 0, "right": 1, "bottom": 1}}); + let (tx, rx) = crossbeam_channel::unbounded(); + let send = |seq: u64, event: &str| { + tx.send(RawNotification { + seq, + recv_wall_us: 1, + recv_mach_ns: seq * 1000, + bytes: format!(r#"{{"event":{event},"state":{{"a":1}}}}"#).into_bytes(), + }) + .expect("send"); + }; + send( + 1, + &serde_json::json!({"type": "FocusChange", "content": ["SystemForeground", window]}) + .to_string(), + ); + // a command that happens to carry a number is not about a window + send(2, r#"{"type":"DebugWindow","content":131072}"#); + drop(tx); + + let mut prober = EventProber::new( + |hwnd| { + Some(WindowFacts { + hwnd, + at_ns: 0, + exists: true, + visible: true, + cloaked: true, + minimized: false, + rect: None, + komorebi_says: None, + }) + }, + |_| None, + ); + write_loop( + &rx, + File::create(dir.join("events.jsonl")).expect("file"), + &dir.join("states"), + &EventCounters::default(), + None, + Some(&mut prober), + ); + + let rows: Vec = std::fs::read_to_string(dir.join("events.jsonl")) + .expect("events") + .lines() + .map(|line| serde_json::from_str(line).expect("row")) + .collect(); + let probe = rows[0].probe.as_ref().expect("the window event was probed"); + assert_eq!((probe.hwnd, probe.cloaked), (131_072, true)); + assert!(probe.at_ns > 0, "stamped when it was read"); + assert!(rows[1].probe.is_none(), "a command is not about a window"); + let _ = std::fs::remove_dir_all(&dir); + } } diff --git a/komorebi-recorder/src/sources/mod.rs b/komorebi-recorder/src/sources/mod.rs index de2cdbf3..e3f52c10 100644 --- a/komorebi-recorder/src/sources/mod.rs +++ b/komorebi-recorder/src/sources/mod.rs @@ -13,6 +13,7 @@ //! | `komorebi_log` | `komorebi.log` (a byte-for-byte slice of komorebi's own log) | both hosts | //! | `oslog` | `oslog/*.ndjson`, one `log stream` per seam (komorebi, the system, each managed app) | macOS | //! | `wintrace` | `oslog/system.ndjson` (`WinEvents`) and `oslog/eventlog.ndjson` (Event Log) | Windows | +//! | `winprobe` | no file of its own: a `probe` key on rows of `events.jsonl` | Windows, with `--probe-windows` | //! | `komorebi_version` | no file: asks komorebi's binary what it is, for the manifest | both hosts | //! | `managed_apps` | no file: decides which processes get a trace of their own | macOS | //! @@ -41,4 +42,5 @@ pub mod komorebi_log; pub mod komorebi_version; pub mod managed_apps; pub mod oslog; +pub mod winprobe; pub mod wintrace; diff --git a/komorebi-recorder/src/sources/winprobe.rs b/komorebi-recorder/src/sources/winprobe.rs new file mode 100644 index 00000000..25e58656 --- /dev/null +++ b/komorebi-recorder/src/sources/winprobe.rs @@ -0,0 +1,481 @@ +//! What Windows itself says about a window at the moment komorebi reports an event about it +//! (`record --probe-windows`). +//! +//! komorebi for Windows hides a window by *cloaking* it where it is, so its `State` cannot say +//! whether a window is really hidden: a window on a workspace that is not showing has the same rect +//! as when it was. The only way to know is to ask Windows. With the probe on, each window event is +//! followed by a read of the window's real state (is it visible, cloaked, minimized, and where is +//! it), stored with the event, and sometimes by komorebi's own answer to "would you manage this +//! window, and why". The analysis then checks the two against each other +//! ([`crate::analysis::invariants::check_probed`]): a window komorebi shows that Windows has +//! hidden, one it hides that Windows shows, and a visible window it does not manage. +//! +//! The probe runs on the thread that writes events, never the one that accepts them: komorebi +//! delivers a notification while holding its own locks, and a probe that asked komorebi something +//! while the accept thread was stuck would wait on a lock that is waiting on it. +//! +//! # What is written +//! +//! No file of its own. A probe is stored as the `probe` key of the `events.jsonl` row of the event +//! it is about ([`EventRow::probe`](crate::sources::komorebi_events::EventRow::probe)), as a +//! [`WindowFacts`]. A row has a probe only for a window event, and only in a recording made with +//! `--probe-windows`; every other row, and every row of any other recording, has none. +//! +//! # Host and flavor +//! +//! The prober reads komorebi for Windows' events (it asks the [`Flavor::Windows`] rules which +//! events are about which window) and Windows' window state, so it is meaningful only for a +//! recording made on Windows. The module builds on every host, because the types are read from +//! sessions anywhere, but on any other host [`EventProber::windows`] has nothing to ask +//! ([`crate::platform::window::inspect`] returns `None`), and `record --probe-windows` is refused +//! there. + +use crate::analysis::events::Family; +use crate::analysis::flavor::Flavor; +use schemars::JsonSchema; +use serde::Deserialize; +use serde::Serialize; +use serde_json::Value; +use std::collections::HashMap; + +/// komorebi's answers are kept to this much of the start: they list the rules that matched, which +/// is what matters, and not every attribute of the window. +const SAYS_LIMIT: usize = 2_000; + +/// A window komorebi is asked about again no sooner than this. +const ASK_AGAIN_NS: u64 = 5_000_000_000; + +/// How many windows are remembered as asked about before the memory is cleared. +const ASKED_CAPACITY: usize = 4096; + +/// Events after which komorebi's decision about a window may have changed, so it is asked again. +const DECIDING_EVENTS: [&str; 5] = ["Show", "Manage", "Unmanage", "Uncloak", "TitleUpdate"]; + +/// What Windows said about one window, read just after komorebi reported an event about it. +/// +/// This is the value of the `probe` key of an `events.jsonl` row. Every field is always written, +/// except `komorebi_says`, which is left out when absent. +/// +/// # Example +/// +/// ```rust +/// use komorebi_recorder::sources::winprobe::WindowFacts; +/// +/// let facts = WindowFacts { +/// hwnd: 131_072, +/// at_ns: 5_000_000_000, +/// exists: true, +/// visible: true, +/// cloaked: true, +/// minimized: false, +/// rect: Some([0, 0, 1920, 1080]), +/// komorebi_says: None, +/// }; +/// +/// // a cloaked window is how komorebi hides one: it is still `visible`, and not shown +/// assert!(!facts.shown()); +/// assert_eq!(facts.hidden_how(), Some("cloaked")); +/// +/// let text = serde_json::to_string(&facts)?; +/// assert!(text.contains(r#""rect":[0,0,1920,1080]"#) && !text.contains("komorebi_says")); +/// assert_eq!(serde_json::from_str::(&text)?, facts); +/// # Ok::<(), Box>(()) +/// ``` +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +pub struct WindowFacts { + /// The window handle, the same number komorebi for Windows gives the window as its `hwnd`. + pub hwnd: u32, + /// When it was read, in nanoseconds on the master clock: later than the event, by however + /// long the event waited to be written. + pub at_ns: u64, + /// The handle still names a window (it does not once the window is destroyed). + pub exists: bool, + /// `IsWindowVisible`. A window that komorebi has hidden by cloaking it is still visible in + /// this sense: see `cloaked`. + pub visible: bool, + /// Cloaked by the shell (on another virtual desktop), by an application, or by komorebi, + /// which is how komorebi for Windows hides a window. + pub cloaked: bool, + /// `IsIconic`. + pub minimized: bool, + /// Where Windows has it, as komorebi reads a window's rect (the visible frame): `[left, top, + /// right, bottom]` in physical pixels. `None` if the window does not exist or its rect could + /// not be read. + pub rect: Option<[i32; 4]>, + /// komorebi's own answer to "would you manage this window, and which rules matched", verbatim + /// and cut short to its first 2,000 characters. Only asked now and then, so usually absent. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub komorebi_says: Option, +} + +impl WindowFacts { + /// Returns `true` if and only if the window is on screen to a person: it exists, is visible, + /// and is neither cloaked nor minimized. + #[must_use] + pub fn shown(&self) -> bool { + self.exists && self.visible && !self.cloaked && !self.minimized + } + + /// Returns how the window is hidden, in words, if it is: `destroyed`, `cloaked`, `minimized` + /// or `not visible`, the first that applies in that order. `None` for a window that is not + /// hidden by any of them. + #[must_use] + pub fn hidden_how(&self) -> Option<&'static str> { + if !self.exists { + Some("destroyed") + } else if self.cloaked { + Some("cloaked") + } else if self.minimized { + Some("minimized") + } else if !self.visible { + Some("not visible") + } else { + None + } + } +} + +/// When komorebi is asked about a window: the first time it is seen, and again after an event that +/// can change the answer, but never in a flurry. +/// +/// The events that can change the answer are `Show`, `Manage`, `Unmanage`, `Uncloak` and +/// `TitleUpdate`, and a window is asked about again after one of them only if the last time was +/// at least five seconds before. The policy remembers up to 4096 windows and forgets all of them +/// when that fills, so a window is then asked about as if it were new. +/// +/// # Example +/// +/// ```rust +/// use komorebi_recorder::sources::winprobe::AskPolicy; +/// +/// let mut policy = AskPolicy::default(); +/// let second = 1_000_000_000; +/// +/// assert!(policy.should_ask(7, "FocusChange", second), "the first time a window is seen"); +/// assert!(!policy.should_ask(7, "FocusChange", 60 * second), "focus does not change komorebi's mind"); +/// assert!(policy.should_ask(7, "Show", 61 * second), "a Show after the quiet period can"); +/// assert!(!policy.should_ask(7, "Show", 62 * second), "but not in a flurry"); +/// assert!(policy.should_ask(8, "FocusChange", 62 * second), "another window is its own first time"); +/// ``` +#[derive(Debug, Default)] +pub struct AskPolicy { + asked: HashMap, +} + +impl AskPolicy { + /// Returns `true` if komorebi should be asked about window `hwnd` now, after an event named + /// `event` heard at `now_ns` (nanoseconds on the master clock), and records that it was. + /// + /// A window not asked about before is always due. After that it is due only on one of the + /// events that can change komorebi's decision, and at least five seconds after the last time. + pub fn should_ask(&mut self, hwnd: u32, event: &str, now_ns: u64) -> bool { + if self.asked.len() >= ASKED_CAPACITY { + self.asked.clear(); + } + + let due = match self.asked.get(&hwnd) { + None => true, + Some(last) => { + DECIDING_EVENTS.contains(&event) && now_ns.saturating_sub(*last) >= ASK_AGAIN_NS + } + }; + if due { + self.asked.insert(hwnd, now_ns); + } + due + } +} + +fn shortened(text: String) -> String { + if text.chars().count() <= SAYS_LIMIT { + return text; + } + + let mut cut: String = text.chars().take(SAYS_LIMIT).collect(); + cut.push('…'); + cut +} + +type Inspect = Box Option + Send>; +type Ask = Box Option + Send>; + +/// Reads the real state of the window a komorebi event is about. +/// +/// A prober is built with two functions, one that reads a window ([`WindowFacts`]) and one that +/// asks komorebi about it, so that the logic (which events are probed, when komorebi is asked) can +/// be tested without Windows or komorebi. [`EventProber::windows`] builds the real one. +/// +/// # Example +/// +/// ```rust +/// use komorebi_recorder::sources::winprobe::{EventProber, WindowFacts}; +/// use serde_json::json; +/// +/// let mut prober = EventProber::new( +/// |hwnd| Some(WindowFacts { +/// hwnd, +/// at_ns: 0, +/// exists: true, +/// visible: true, +/// cloaked: false, +/// minimized: false, +/// rect: Some([0, 0, 100, 100]), +/// komorebi_says: None, +/// }), +/// |_hwnd| Some("would manage: yes".to_string()), +/// ); +/// +/// let window = json!({"hwnd": 131072, "title": "t", "exe": "a.exe", "class": "C", +/// "rect": {"left": 0, "top": 0, "right": 1, "bottom": 1}}); +/// let event = json!({"type": "FocusChange", "content": ["SystemForeground", window]}); +/// +/// let facts = prober.probe(&event, 5_000).ok_or("not probed")?; +/// assert_eq!((facts.hwnd, facts.at_ns), (131_072, 5_000), "stamped when it was read"); +/// assert!(facts.shown()); +/// assert_eq!(facts.komorebi_says.as_deref(), Some("would manage: yes"), "the first time it is seen"); +/// +/// // a command, even one that carries a number, is not about a window +/// assert_eq!(prober.probe(&json!({"type": "DebugWindow", "content": 131072}), 6_000), None); +/// # Ok::<(), Box>(()) +/// ``` +pub struct EventProber { + inspect: Inspect, + ask: Ask, + policy: AskPolicy, +} + +impl EventProber { + /// Returns a prober that reads windows with `inspect` and asks komorebi with `ask`. + /// + /// Both are given a window handle. `inspect` returns `None` for a window it cannot read, and + /// `ask` returns komorebi's answer, or `None` for none. + pub fn new( + inspect: impl FnMut(u32) -> Option + Send + 'static, + ask: impl FnMut(u32) -> Option + Send + 'static, + ) -> Self { + Self { + inspect: Box::new(inspect), + ask: Box::new(ask), + policy: AskPolicy::default(), + } + } + + /// Returns the real prober: Win32 for the window ([`crate::platform::window::inspect`]), + /// komorebi's socket for its verdict (a `DebugWindow` query, see [`crate::komorebi_link`]). + /// + /// komorebi's reply is cut to its first 2,000 characters; an empty reply, and a query that + /// fails, are no answer. On any host but Windows there is nothing to read, so every probe is + /// `None`. + /// + /// This talks to Windows and to komorebi, so it has no doctest and no unit test: the logic it + /// drives is covered through [`EventProber::new`] by the tests of this file, and the Win32 + /// reads are checked on a real machine (see `docs/dev/windows-checkpoints.md`). + #[must_use] + pub fn windows() -> Self { + Self::new(crate::platform::window::inspect, |hwnd| { + let reply = + crate::komorebi_link::send_query(&crate::komorebi_link::Message::DebugWindow(hwnd)) + .ok()?; + (!reply.trim().is_empty()).then(|| shortened(reply)) + }) + } + + /// Probes the window `event` (the JSON komorebi sent) is about, if it is about one, and returns + /// what was read, with `at_ns` set to `now_ns` (the time on the master clock). + /// + /// Only a window-manager event is: a query or a command may carry a number, and one of them, + /// the `DebugWindow` this asks, is itself reported to subscribers, which must not set off + /// another probe. The result is `None` for any other event, for an event with no window + /// handle, and when the window cannot be read. komorebi is asked about the window only if it + /// still exists and the [`AskPolicy`] says it is due. + pub fn probe(&mut self, event: &Value, now_ns: u64) -> Option { + let info = Flavor::Windows.rules().describe_event(event); + if info.family != Family::WindowManager { + return None; + } + + let hwnd = info.window_id?; + let mut facts = (self.inspect)(hwnd)?; + facts.at_ns = now_ns; + if facts.exists && self.policy.should_ask(hwnd, &info.name, now_ns) { + facts.komorebi_says = (self.ask)(hwnd); + } + + Some(facts) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + use std::sync::Arc; + use std::sync::atomic::AtomicUsize; + use std::sync::atomic::Ordering; + + fn facts(hwnd: u32) -> WindowFacts { + WindowFacts { + hwnd, + at_ns: 0, + exists: true, + visible: true, + cloaked: false, + minimized: false, + rect: Some([0, 0, 100, 100]), + komorebi_says: None, + } + } + + fn window(hwnd: i64) -> Value { + json!({"hwnd": hwnd, "title": "t", "exe": "a.exe", "class": "C", + "rect": {"left": 0, "top": 0, "right": 1, "bottom": 1}}) + } + + fn event(name: &str, hwnd: i64) -> Value { + json!({"type": name, "content": ["ObjectShow", window(hwnd)]}) + } + + struct Counting { + inspected: Arc, + asked: Arc, + prober: EventProber, + } + + fn counting() -> Counting { + let (inspected, asked) = (Arc::new(AtomicUsize::new(0)), Arc::new(AtomicUsize::new(0))); + let (i, a) = (inspected.clone(), asked.clone()); + Counting { + inspected, + asked, + prober: EventProber::new( + move |hwnd| { + i.fetch_add(1, Ordering::Relaxed); + Some(facts(hwnd)) + }, + move |_| { + a.fetch_add(1, Ordering::Relaxed); + Some("would manage: yes".to_string()) + }, + ), + } + } + + #[test] + fn a_window_event_is_followed_by_a_read_of_that_window_stamped_when_it_was_read() { + let mut probe = counting(); + let found = probe + .prober + .probe(&event("FocusChange", 131_072), 5_000) + .expect("facts"); + assert_eq!((found.hwnd, found.at_ns), (131_072, 5_000)); + assert!(found.shown()); + assert_eq!(probe.inspected.load(Ordering::Relaxed), 1); + } + + #[test] + fn what_is_not_a_window_event_is_never_probed() { + let mut probe = counting(); + // a command with a number (the shape of the DebugWindow this probe itself sends), a + // notification about the machine, a state query, and nonsense + for not_a_window in [ + json!({"type": "DebugWindow", "content": 131_072}), + json!({"type": "FocusWorkspaceNumber", "content": 2}), + json!({"type": "SessionLocked"}), + json!({"type": "State"}), + json!("LeftAssociatedVirtualDesktop"), + json!(12), + ] { + assert_eq!(probe.prober.probe(¬_a_window, 1), None, "{not_a_window}"); + } + assert_eq!(probe.inspected.load(Ordering::Relaxed), 0); + assert_eq!(probe.asked.load(Ordering::Relaxed), 0, "no feedback loop"); + } + + #[test] + fn komorebi_is_asked_about_a_window_the_first_time_and_then_after_events_that_can_change_its_mind() + { + let mut probe = counting(); + let asked = |probe: &Counting| probe.asked.load(Ordering::Relaxed); + + let first = probe + .prober + .probe(&event("FocusChange", 7), 1) + .expect("facts"); + assert_eq!(first.komorebi_says.as_deref(), Some("would manage: yes")); + assert_eq!(asked(&probe), 1); + + // focus moving to it again says nothing new, however long it has been + probe.prober.probe(&event("FocusChange", 7), 60_000_000_000); + assert_eq!(asked(&probe), 1); + + // a Show after the quiet period can: asked again + probe.prober.probe(&event("Show", 7), 61_000_000_000); + assert_eq!(asked(&probe), 2); + + // but not in a flurry + probe.prober.probe(&event("Show", 7), 62_000_000_000); + assert_eq!(asked(&probe), 2); + + // and another window is its own first time + probe.prober.probe(&event("FocusChange", 8), 62_000_000_000); + assert_eq!(asked(&probe), 3); + } + + #[test] + fn a_destroyed_window_is_recorded_as_gone_and_komorebi_is_not_asked_about_it() { + let mut probe = EventProber::new( + |hwnd| { + Some(WindowFacts { + exists: false, + visible: false, + rect: None, + ..facts(hwnd) + }) + }, + |_| panic!("a destroyed window is not asked about"), + ); + let found = probe.probe(&event("Destroy", 9), 1).expect("facts"); + assert_eq!(found.hidden_how(), Some("destroyed")); + } + + #[test] + fn how_a_window_is_hidden_is_named_and_a_shown_one_is_not() { + let mut window = facts(1); + assert!(window.shown()); + assert_eq!(window.hidden_how(), None); + + window.cloaked = true; + assert!(!window.shown()); + assert_eq!(window.hidden_how(), Some("cloaked")); + + window.cloaked = false; + window.minimized = true; + assert_eq!(window.hidden_how(), Some("minimized")); + + window.minimized = false; + window.visible = false; + assert_eq!(window.hidden_how(), Some("not visible")); + } + + #[test] + fn the_facts_roundtrip_and_old_rows_without_them_still_parse() { + let mut with = facts(5); + with.komorebi_says = Some("rules matched".to_string()); + let text = serde_json::to_string(&with).expect("json"); + assert_eq!( + serde_json::from_str::(&text).expect("parse"), + with + ); + + let without = serde_json::to_string(&facts(5)).expect("json"); + assert!(!without.contains("komorebi_says")); + } + + #[test] + fn a_long_answer_is_cut_short() { + let cut = shortened("x".repeat(SAYS_LIMIT * 2)); + assert_eq!(cut.chars().count(), SAYS_LIMIT + 1); + assert!(cut.ends_with('…')); + assert_eq!(shortened("short".to_string()), "short"); + } +} diff --git a/komorebi-recorder/src/testkit.rs b/komorebi-recorder/src/testkit.rs index 9f3b72f9..d554cd78 100644 --- a/komorebi-recorder/src/testkit.rs +++ b/komorebi-recorder/src/testkit.rs @@ -57,6 +57,7 @@ struct PendingEvent { video_s: f64, event: Value, state: Option, + probe: Option, } /// A session under construction. Times are seconds of video time. @@ -151,10 +152,21 @@ impl SessionBuilder { video_s, event, state, + probe: None, }); self } + /// What Windows said about the window of the event added last, as a recording made with + /// `--probe-windows` stores it. + #[must_use] + pub fn probe(mut self, facts: crate::sources::winprobe::WindowFacts) -> Self { + if let Some(last) = self.events.last_mut() { + last.probe = Some(facts); + } + self + } + /// One line of komorebi's plaintext log at `video_s`, e.g. `log(2.0, "INFO", "komorebi::x: hello")`. #[must_use] pub fn log(mut self, video_s: f64, level: &str, message: &str) -> Self { @@ -263,6 +275,7 @@ impl SessionBuilder { state_id: (state_id > 0).then_some(state_id), parse_error: None, raw: None, + probe: pending.probe.clone(), }; rows.push_str(&serde_json::to_string(&row).expect("row")); rows.push('\n'); diff --git a/komorebi-recorder/tests/windows_flavor.rs b/komorebi-recorder/tests/windows_flavor.rs index 055c6d90..ebede8e6 100644 --- a/komorebi-recorder/tests/windows_flavor.rs +++ b/komorebi-recorder/tests/windows_flavor.rs @@ -171,3 +171,113 @@ fn a_mac_session_does_not_get_that_warning() { "{warnings:?}" ); } + +/// A recording made with `--probe-windows` of komorebi showing a window that Windows has cloaked. +fn probed_session(name: &str) -> String { + use komorebi_recorder::sources::winprobe::WindowFacts; + + let state = windows_state( + 65_537, + vec![vec![windows_window( + 131_072, + "chrome.exe", + Some([0, 0, 960, 1040]), + )]], + ); + let cloaked = |at_ns| WindowFacts { + hwnd: 131_072, + at_ns, + exists: true, + visible: true, + cloaked: true, + minimized: false, + rect: Some([0, 0, 960, 1040]), + komorebi_says: Some("would manage: yes\n matched: none".to_string()), + }; + let focus = || { + json!({"type": "FocusChange", "content": ["SystemForeground", + windows_window(131_072, "chrome.exe", Some([0, 0, 960, 1040]))]}) + }; + + let (session, _) = SessionBuilder::new(name) + .flavor(Flavor::Windows) + .event(2.0, focus(), Some(state.clone())) + .probe(cloaked(komorebi_recorder::testkit::mach_ns(2.01))) + .event(4.0, focus(), Some(state)) + .probe(cloaked(komorebi_recorder::testkit::mach_ns(4.01))) + .build(); + session.dir.display().to_string() +} + +#[test] +fn a_window_komorebi_shows_that_windows_cloaked_is_a_finding_in_the_overview() { + let dir = probed_session("windows-flavor-probed-overview"); + let answer = run(&["overview", &dir, "--json"]); + let findings = answer["data"]["findings"].as_array().expect("findings"); + let found = findings + .iter() + .find(|finding| finding["kind"] == "invariant:visible-window-hidden") + .unwrap_or_else(|| panic!("no such finding in {findings:#?}")); + + assert_eq!(found["severity"], "error"); + assert!( + found["summary"] + .as_str() + .is_some_and(|text| text.contains("cloaked")), + "{found}" + ); + assert_eq!(found["window_ids"][0], 131_072); +} + +#[test] +fn a_trace_shows_what_windows_said_and_what_that_says_about_komorebis_state() { + let dir = probed_session("windows-flavor-probed-trace"); + let trace = run(&["trace", &dir, "event:1", "--json"]); + let probe = &trace["data"]["window_probe"]; + assert_eq!(probe["facts"]["cloaked"], true); + assert_eq!(probe["violations"][0]["kind"], "visible-window-hidden"); + + let text = Command::new(env!("CARGO_BIN_EXE_komorebi-recorder")) + .args(["trace", &dir, "event:1"]) + .output() + .expect("run"); + let text = String::from_utf8_lossy(&text.stdout); + assert!( + text.contains("what Windows said about window 131072"), + "{text}" + ); + assert!(text.contains("cloaked true"), "{text}"); + assert!(text.contains("komorebi says:"), "{text}"); +} + +#[test] +fn a_recording_without_probes_says_nothing_about_them() { + let dir = session("windows-flavor-unprobed"); + let trace = run(&["trace", &dir, "event:1", "--json"]); + assert!(trace["data"].get("window_probe").is_none(), "{trace}"); +} + +#[cfg(not(windows))] +#[test] +fn a_host_that_is_not_windows_refuses_to_probe_windows_before_it_records_anything() { + let out = std::env::temp_dir().join(format!("komorebi-probe-refused-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&out); + + let output = Command::new(env!("CARGO_BIN_EXE_komorebi-recorder")) + .args([ + "record", + "--probe-windows", + "--no-video", + "--duration", + "1", + "--out", + ]) + .arg(&out) + .output() + .expect("run"); + + assert!(!output.status.success()); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!(stderr.contains("not Windows"), "{stderr}"); + assert!(!out.exists(), "no session was started"); +}