From 13713d65e82d61f1ec963993e2eceec23fea29aa Mon Sep 17 00:00:00 2001 From: LGUG2Z Date: Wed, 28 Oct 2026 16:10:41 -0700 Subject: [PATCH] feat(assert): scripted pass/fail checks over a recording's findings, log, motion and notes assert --check " " evaluates checks with the existing analyses (findings, logq, motion, blank, annotations), one Check trait and one registry line per kind. Exit 6 (ExitKind::Assertion) when a check fails; checks that cannot be measured are errors (2/3/4), never a pass. JSON envelope assert/1. --- README.md | 5 +- docs/usage/investigating-a-recording.md | 3 +- docs/usage/recording-a-bug-repro.md | 25 + komorebi-recorder/src/analysis/assertions.rs | 1244 ++++++++++++++++++ komorebi-recorder/src/analysis/envelope.rs | 55 +- komorebi-recorder/src/analysis/mod.rs | 1 + komorebi-recorder/src/analysis/schemas.rs | 5 + komorebi-recorder/src/cli/assert.rs | 63 + komorebi-recorder/src/cli/mod.rs | 10 + komorebi-recorder/src/guide.md | 14 +- komorebi-recorder/tests/assert.rs | 349 +++++ 11 files changed, 1768 insertions(+), 6 deletions(-) create mode 100644 komorebi-recorder/src/analysis/assertions.rs create mode 100644 komorebi-recorder/src/cli/assert.rs create mode 100644 komorebi-recorder/tests/assert.rs diff --git a/README.md b/README.md index 564725d9..e4c18697 100644 --- a/README.md +++ b/README.md @@ -63,7 +63,10 @@ komorebi-recorder trace ./repro event:7 # one event, from the handler that ran ``` A script that drives the repro can name its steps on the timeline while recording, with -`komorebi-recorder mark "step 3: move window right"`; they show up as notes. +`komorebi-recorder mark "step 3: move window right"`; they show up as notes. And the same script can end with a +pass or fail: `komorebi-recorder assert ./repro --check "findings kind=layout-mismatch count<=0" --check "log \"level=ERROR\" count==0"` +exits 0 when every check holds and 6 when one does not (checks over findings, the log, motion, blank regions and +notes; see the guide). For example, a four second recording of a Mac with komorebi running: diff --git a/docs/usage/investigating-a-recording.md b/docs/usage/investigating-a-recording.md index 1d10e4dc..66246e39 100644 --- a/docs/usage/investigating-a-recording.md +++ b/docs/usage/investigating-a-recording.md @@ -14,7 +14,7 @@ komorebi-recorder guide # the playbook, for an agent Every command takes `--json`, which prints an envelope `{ok, schema, data, warnings, truncated, next_cursor, error}`; `komorebi-recorder schema` lists the schemas and `komorebi-recorder schema brief/1` prints one. Exit codes: 0 ok, 1 error, 2 usage, 3 not found, 4 the session cannot answer (still recording, or the recorder died: `reindex` it), 5 over -budget. +budget, 6 an `assert` check did not hold. ## The viewer @@ -127,6 +127,7 @@ and `event:12.end` when it finished. | `framediff A B [--region ..] [--png FILE] [--against SESSION]` | How two frames differ: changed share, mean and peak brightness delta, the box of change, and a PNG with the changed pixels in red | | `blank --region x,y,w,h` | When a region was blank (one colour, named: `white`, `black`, a hex code) or frozen (identical to the frame before): "the window went white for 200 ms" | | `compare A B` | Two recordings side by side: duration, frames, effective fps, worst frame gap, motion, findings by kind, annotations, and the difference | +| `assert --check "KIND ARGS METRIC<=N"` | Pass/fail checks for a scripted scenario, repeatable. Kinds: `findings kind=K [severity=S] count<=N`, `log "" count==N`, `motion [region=x,y,w,h] [from=T] [to=T] segments\|active_ms\|peak_share<=N`, `blank region=x,y,w,h [from=T] [to=T] [color=white] ms<=N`, `annotations kind=note\|bug\|finding count>=N`. Operators `== != <= >= < >`. Prints `PASS`/`FAIL` per check; exit 0 if all pass, 6 if one fails, 4 if one cannot be measured (no video), 2 if malformed. `--json` gives `assert/1` | | `annotate`, `annotations` | Add, change and list bug regions, notes and findings | | `mark TEXT` | While recording: drop a named point note (author `script`) onto the timeline from a script; it then appears in `annotations`, `brief` and the viewer like any note | diff --git a/docs/usage/recording-a-bug-repro.md b/docs/usage/recording-a-bug-repro.md index 14aea578..1f041be4 100644 --- a/docs/usage/recording-a-bug-repro.md +++ b/docs/usage/recording-a-bug-repro.md @@ -144,6 +144,31 @@ It marks the session named by `--session DIR`, else by the environment variable still running. A recording that has finished is refused: it has no "now" to mark; use `annotate add` with a time instead. `--json` prints the `mark/1` envelope. +### Checking the result from a script + +`assert` turns the recording into a pass or fail, so the same script that reproduces the bug can end by saying whether +it happened. The exit status is the result: 0 when every check holds, 6 when one does not, 4 when one could not be +measured (for example `motion` on a recording without video) and 2 when a check is malformed. + +``` +komorebi-recorder record --until-exit --out ./repro -- ./repro.sh # repro.sh calls `komorebi-recorder mark` between steps +komorebi-recorder assert ./repro --check "annotations kind=note count>=3" --check "findings kind=layout-mismatch count<=0" --check "log \"level=ERROR\" count==0" --check "blank region=0,0,800,600 color=white ms<=100" +``` + +``` +PASS annotations kind=note count>=3 (observed 4) +FAIL findings kind=layout-mismatch count<=0 (observed 2) + f-3a9c1e invariant:layout-mismatch at 00:03.280: ... +PASS log "level=ERROR" count==0 (observed 0) +PASS blank region=0,0,800,600 color=white ms<=100 (observed 0) +1 of 4 check(s): 1 failed +``` + +The first check confirms the script's `mark`s were recorded. A check is a kind, its arguments and a comparison +(`== != <= >= < >`); the kinds are `findings`, `log` (a `logq` query), `motion`, `blank` and `annotations`, and +times in `from=` and `to=` are any time the other commands take, including `event:N` and `bug:`. `--json` prints the +`assert/1` envelope. `assert` only reads the session. + ### Without video `komorebi-recorder record --no-video` records everything but the screen: komorebi's events and log, the OS trace and diff --git a/komorebi-recorder/src/analysis/assertions.rs b/komorebi-recorder/src/analysis/assertions.rs new file mode 100644 index 00000000..cbc4141d --- /dev/null +++ b/komorebi-recorder/src/analysis/assertions.rs @@ -0,0 +1,1244 @@ +//! Scripted checks over a recording: the engine of `assert`. +//! +//! A script that drives komorebi while it is recorded ends up wanting a yes or no: did a layout +//! mismatch show up, did the log stay free of errors, did the picture settle? `assert` answers with +//! its exit code, by evaluating *checks* with the analyses the other commands use. There is no +//! komorebi-specific language here: a check is a kind and what to measure, and a comparison. +//! +//! ```text +//! +//! ``` +//! +//! `` is one of `==`, `!=`, `<=`, `>=`, `<`, `>`. The arguments are `key=value` words +//! separated by spaces; a value with spaces is quoted with `"` or `'`. The kinds are listed in +//! [`KINDS`]: +//! +//! | Kind | Arguments | Metrics | +//! |---|---|---| +//! | `findings` | `kind=K`, `severity=error\|warn\|info` | `count` | +//! | `log` | a [`logq`](crate::analysis::logq) query, as written for `logq` (optionally quoted) | `count` | +//! | `motion` | `region=x,y,w,h`, `from=T`, `to=T` | `segments`, `active_ms`, `peak_share`, `peak_delta` | +//! | `blank` | `region=x,y,w,h` (required), `from=T`, `to=T`, `color=white` | `ms` (the longest stretch), `total_ms`, `stretches` | +//! | `annotations` | `kind=note\|bug\|finding` | `count` | +//! +//! `T` is any time `state --at` takes: video time, `logtime:HH:MM:SS.mmm`, `event:N`, `bug:`, +//! offsets of those. +//! +//! A check ends as [`Status::Pass`], [`Status::Fail`] (it was measured and the comparison does +//! not hold) or [`Status::Error`] (it could not be measured: no video, a host that cannot decode +//! it, a time that does not exist). A malformed check is a usage error before anything is +//! measured. [`AssertData::exit_kind`] turns the lot into the exit code: errors are never +//! counted as a pass. +//! +//! Adding a kind is one struct that implements [`Check`] and one line in [`KINDS`]. +//! +//! # Example +//! +//! ```rust +//! use komorebi_recorder::analysis::Analysis; +//! use komorebi_recorder::analysis::assertions::{self, Status}; +//! use komorebi_recorder::testkit::SessionBuilder; +//! +//! let (session, _) = SessionBuilder::new("doc-assert") +//! .log(2.0, "ERROR", "komorebi::window: could not place window 4711") +//! .build(); +//! let analysis = Analysis::open(&session.dir)?; +//! +//! let data = assertions::run_all( +//! &analysis, +//! &[ +//! "log \"level=ERROR\" count==0".to_string(), +//! "annotations kind=note count>=0".to_string(), +//! ], +//! )?; +//! assert_eq!(data.checks[0].status, Status::Fail); +//! assert_eq!(data.checks[0].observed, serde_json::json!(1)); +//! assert_eq!(data.checks[1].status, Status::Pass); +//! assert_eq!(data.exit_kind().code(), 6); +//! # Ok::<(), Box>(()) +//! ``` + +use super::Analysis; +use super::blank::DEFAULT_MIN_DURATION_S; +use super::blank::DEFAULT_VARIANCE; +use super::blank::StretchKind; +use super::budget::Budget; +use super::envelope; +use super::envelope::ExitKind; +use super::findings::FindingsQuery; +use super::invariants::Severity; +use super::logq::LogqQuery; +use super::motion::ScanOptions; +use super::pixels::Crop; +use super::video::SessionVideo; +use crate::annotations; +use crate::timecode; +use color_eyre::eyre; +use regex::Regex; +use schemars::JsonSchema; +use serde::Serialize; +use serde_json::Value; +use std::collections::BTreeMap; +use std::fmt; +use std::fmt::Write; +use std::sync::LazyLock; + +/// The name that appears in the `schema` of `assert --json`. +pub const SCHEMA_ASSERT: &str = "assert/1"; + +/// How an observed number is compared with the one in the check. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Op { + /// `==` + Eq, + /// `!=` + Ne, + /// `<=` + Le, + /// `>=` + Ge, + /// `<` + Lt, + /// `>` + Gt, +} + +impl Op { + fn parse(text: &str) -> Option { + Some(match text { + "==" => Self::Eq, + "!=" => Self::Ne, + "<=" => Self::Le, + ">=" => Self::Ge, + "<" => Self::Lt, + ">" => Self::Gt, + _ => return None, + }) + } + + /// Returns whether `observed expected` holds. + #[must_use] + pub fn holds(self, observed: f64, expected: f64) -> bool { + match self { + Self::Eq => (observed - expected).abs() < f64::EPSILON, + Self::Ne => (observed - expected).abs() >= f64::EPSILON, + Self::Le => observed <= expected, + Self::Ge => observed >= expected, + Self::Lt => observed < expected, + Self::Gt => observed > expected, + } + } +} + +impl fmt::Display for Op { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(match self { + Self::Eq => "==", + Self::Ne => "!=", + Self::Le => "<=", + Self::Ge => ">=", + Self::Lt => "<", + Self::Gt => ">", + }) + } +} + +/// The comparison that ends a check: `count<=0`, `active_ms>=250`, `peak_share<0.2`. +#[derive(Debug, Clone, PartialEq)] +pub struct Comparison { + /// What is measured: `count`, `segments`, `ms`, ... + pub metric: String, + /// How it is compared. + pub op: Op, + /// What it is compared with. + pub value: f64, +} + +impl Comparison { + /// Returns whether the comparison holds for this measurement. + #[must_use] + pub fn holds(&self, observed: f64) -> bool { + self.op.holds(observed, self.value) + } + + fn require(&self, kind: &str, metrics: &[&str]) -> eyre::Result<()> { + if metrics.contains(&self.metric.as_str()) { + return Ok(()); + } + Err(envelope::usage(format!( + "a `{kind}` check measures {}, not `{}`", + metrics.join(", "), + self.metric + ))) + } +} + +impl fmt::Display for Comparison { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}{}{}", self.metric, self.op, number_text(self.value)) + } +} + +/// What a check kind is given: the text between the kind and the comparison, and the comparison. +#[derive(Debug, Clone)] +pub struct CheckArgs { + /// The arguments as written, without the kind and without the comparison. + pub text: String, + /// The comparison at the end of the check. + pub comparison: Comparison, +} + +impl CheckArgs { + /// Splits the arguments into `key=value` pairs, honouring `"` and `'` quotes around a value or + /// a whole word. + /// + /// # Errors + /// + /// Returns a usage error for a word that is not `key=value`, a key that is not in `allowed`, + /// a key given twice or a quote that is never closed. + pub fn pairs(&self, allowed: &[&str]) -> eyre::Result> { + let mut pairs = BTreeMap::new(); + for word in words(&self.text)? { + let (key, value) = word + .split_once('=') + .filter(|(key, _)| !key.is_empty()) + .ok_or_else(|| { + envelope::usage(format!( + "`{word}` is not key=value (keys: {})", + allowed.join(", ") + )) + })?; + if !allowed.contains(&key) { + return Err(envelope::usage(format!( + "unknown argument `{key}` (arguments: {})", + allowed.join(", ") + ))); + } + if pairs.insert(key.to_string(), value.to_string()).is_some() { + return Err(envelope::usage(format!("`{key}` is given twice"))); + } + } + Ok(pairs) + } +} + +/// Splits `text` into words on whitespace; a quote (`"` or `'`) groups what is inside it, and a +/// backslash in quotes escapes the quote character. +fn words(text: &str) -> eyre::Result> { + let mut out = Vec::new(); + let mut current = String::new(); + let mut started = false; + let mut quote: Option = None; + let mut chars = text.chars().peekable(); + + while let Some(c) = chars.next() { + match quote { + Some(q) if c == q => quote = None, + Some(q) if c == '\\' && chars.peek() == Some(&q) => { + current.push(q); + chars.next(); + } + Some(_) => current.push(c), + None if c == '"' || c == '\'' => { + quote = Some(c); + started = true; + } + None if c.is_whitespace() => { + if started { + out.push(std::mem::take(&mut current)); + started = false; + } + } + None => { + current.push(c); + started = true; + } + } + } + if quote.is_some() { + return Err(envelope::usage(format!( + "a quote is never closed in `{text}`" + ))); + } + if started { + out.push(current); + } + Ok(out) +} + +/// Removes one pair of quotes that wrap the whole text, if there is one. +fn unwrap_quoted(text: &str) -> String { + let text = text.trim(); + let Some(q) = text.chars().next().filter(|c| *c == '"' || *c == '\'') else { + return text.to_string(); + }; + + let mut out = String::new(); + let mut chars = text[1..].chars().peekable(); + while let Some(c) = chars.next() { + if c == '\\' && chars.peek() == Some(&q) { + out.push(q); + chars.next(); + } else if c == q { + // the quote closes: it wraps the whole text only if nothing follows + return if chars.peek().is_none() { + out + } else { + text.to_string() + }; + } else { + out.push(c); + } + } + text.to_string() +} + +static TAIL: LazyLock = LazyLock::new(|| { + Regex::new(r"(?s)^(.*?)\s*\b([A-Za-z_]+)\s*(==|!=|<=|>=|<|>)\s*(-?[0-9]+(?:\.[0-9]+)?)\s*$") + .expect("the comparison pattern is valid") +}); + +fn number_text(value: f64) -> String { + if value.fract() == 0.0 && value.abs() < 1e15 { + format!("{value:.0}") + } else { + let text = format!("{value:.3}"); + text.trim_end_matches('0').trim_end_matches('.').to_string() + } +} + +fn number(value: f64) -> Value { + if value.fract() == 0.0 && value.abs() < 1e15 { + Value::from(value as i64) + } else { + serde_json::Number::from_f64(value).map_or(Value::Null, Value::Number) + } +} + +/// What a check measured. +#[derive(Debug, Clone, PartialEq)] +pub struct CheckOutcome { + /// The number that was compared, as JSON. + pub observed: Value, + /// Whether the comparison holds. + pub passed: bool, + /// What to look at when it does not: the findings, log lines or segments behind the number. + pub detail: Option, +} + +impl CheckOutcome { + fn compare( + comparison: &Comparison, + observed: f64, + detail: impl FnOnce() -> Option, + ) -> Self { + let passed = comparison.holds(observed); + Self { + observed: number(observed), + passed, + detail: if passed { None } else { detail() }, + } + } +} + +/// One kind of check. Implement it and add a line to [`KINDS`]. +pub trait Check: Sized { + /// The word a check starts with. + const NAME: &'static str; + /// One line on what the kind checks and its arguments, listed in a usage error. + const ABOUT: &'static str; + + /// Reads the arguments of a check of this kind. + /// + /// # Errors + /// + /// Returns a usage error for an argument or a metric this kind does not have, or a value that + /// does not parse. + fn parse(args: &CheckArgs) -> eyre::Result; + + /// Measures the recording and compares. + /// + /// # Errors + /// + /// Returns an error when the recording cannot be measured (it recorded no video, the host + /// cannot decode it, a time does not exist); the caller reports it as [`Status::Error`]. + fn run(&self, analysis: &Analysis) -> eyre::Result; +} + +/// A parsed check of any kind, ready to run. +pub trait RunCheck { + /// See [`Check::run`]. + /// + /// # Errors + /// + /// As [`Check::run`]. + fn run(&self, analysis: &Analysis) -> eyre::Result; +} + +impl RunCheck for T { + fn run(&self, analysis: &Analysis) -> eyre::Result { + Check::run(self, analysis) + } +} + +/// A kind of check, as listed in [`KINDS`]. +pub struct CheckKind { + /// The word a check starts with. + pub name: &'static str, + /// One line on the kind and its arguments. + pub about: &'static str, + parse: fn(&CheckArgs) -> eyre::Result>, +} + +const fn kind() -> CheckKind { + CheckKind { + name: T::NAME, + about: T::ABOUT, + parse: |args| Ok(Box::new(T::parse(args)?)), + } +} + +/// Every kind of check. +pub const KINDS: &[CheckKind] = &[ + kind::(), + kind::(), + kind::(), + kind::(), + kind::(), +]; + +fn kinds_listing() -> String { + KINDS + .iter() + .map(|kind| format!(" {}", kind.about)) + .collect::>() + .join("\n") +} + +/// A check that was read: the text it was written as, and what runs it. +pub struct Assertion { + /// The check as written (surrounding whitespace removed). + pub text: String, + check: Box, +} + +impl Assertion { + /// Reads a check. + /// + /// # Errors + /// + /// Returns a usage error, listing the valid kinds, if the kind is unknown, the comparison is + /// missing or malformed, or the kind rejects its arguments. + /// + /// # Example + /// + /// ```rust + /// use komorebi_recorder::analysis::assertions::Assertion; + /// + /// assert!(Assertion::parse("findings kind=log-error count<=0").is_ok()); + /// + /// let error = Assertion::parse("frames count==1").err().expect("unknown kind"); + /// assert!(error.to_string().contains("unknown check kind `frames`")); + /// assert!(error.to_string().contains("findings")); + /// assert!(Assertion::parse("findings count=1").is_err()); + /// ``` + pub fn parse(text: &str) -> eyre::Result { + let text = text.trim(); + let (name, rest) = text.split_once(char::is_whitespace).unwrap_or((text, "")); + + let Some(entry) = KINDS.iter().find(|kind| kind.name == name) else { + return Err(envelope::usage(format!( + "unknown check kind `{name}` in `{text}`; the kinds are:\n{}", + kinds_listing() + ))); + }; + + let Some(captures) = TAIL.captures(rest) else { + return Err(envelope::usage(format!( + "`{text}` does not end in a comparison like `count<=0` (operators: == != <= >= < >); {}", + entry.about + ))); + }; + let comparison = Comparison { + metric: captures[2].to_string(), + op: Op::parse(&captures[3]).expect("the pattern only matches operators"), + value: captures[4] + .parse() + .map_err(|_| envelope::usage(format!("`{}` is not a number", &captures[4])))?, + }; + + let args = CheckArgs { + text: captures[1].to_string(), + comparison, + }; + let check = (entry.parse)(&args).map_err(|error| { + let message = format!("in `{text}`: {error}"); + error_with_context(error, &message) + })?; + Ok(Self { + text: text.to_string(), + check, + }) + } + + /// Runs the check; one that cannot be measured is [`Status::Error`], never a pass. + #[must_use] + pub fn run(&self, analysis: &Analysis) -> CheckResult { + match self.check.run(analysis) { + Ok(outcome) => CheckResult { + check: self.text.clone(), + status: if outcome.passed { + Status::Pass + } else { + Status::Fail + }, + passed: outcome.passed, + observed: outcome.observed, + detail: outcome.detail, + error_kind: None, + }, + Err(error) => { + let kind = match envelope::exit_kind(&error) { + ExitKind::Error => ExitKind::Unusable, + kind => kind, + }; + CheckResult { + check: self.text.clone(), + status: Status::Error, + passed: false, + observed: Value::Null, + detail: Some(error.to_string()), + error_kind: Some(kind), + } + } + } + } +} + +/// Keeps a usage error a usage error while saying which check it came from. +fn error_with_context(error: eyre::Report, message: &str) -> eyre::Report { + match envelope::exit_kind(&error) { + ExitKind::Usage => envelope::usage(message), + _ => error, + } +} + +/// How a check ended. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, JsonSchema)] +#[serde(rename_all = "lowercase")] +pub enum Status { + /// Measured, and the comparison holds. + Pass, + /// Measured, and the comparison does not hold. + Fail, + /// Could not be measured. + Error, +} + +/// The result of one check. +#[derive(Debug, Clone, PartialEq, Serialize, JsonSchema)] +pub struct CheckResult { + /// The check as written. + pub check: String, + /// `pass`, `fail` or `error`. + pub status: Status, + /// Whether the check passed; `false` for `fail` and for `error`. + pub passed: bool, + /// The number that was measured; `null` for `error`. + #[schemars(with = "Option")] + pub observed: Value, + /// For a failure, what is behind the number; for an error, why it could not be measured. + pub detail: Option, + /// For an error, the kind of failure (`unusable`, `not-found`, `usage`, `budget`). + pub error_kind: Option, +} + +/// The `data` of `assert --json`. +#[derive(Debug, Clone, PartialEq, Serialize, JsonSchema)] +pub struct AssertData { + /// Whether every check passed. + pub passed: bool, + /// How many checks failed. + pub failed: usize, + /// How many checks could not be measured. + pub errors: usize, + /// One result per check, in the order they were given. + pub checks: Vec, +} + +impl AssertData { + /// The exit kind of the run: [`ExitKind::Ok`] if everything passed, else the kind of the first + /// check that could not be measured, else [`ExitKind::Assertion`]. + #[must_use] + pub fn exit_kind(&self) -> ExitKind { + if let Some(kind) = self.checks.iter().find_map(|check| check.error_kind) { + kind + } else if self.failed > 0 { + ExitKind::Assertion + } else { + ExitKind::Ok + } + } + + /// A sentence for the summary line and an envelope's error. + #[must_use] + pub fn summary(&self) -> String { + let total = self.checks.len(); + if self.passed { + return format!("all {total} check(s) passed"); + } + let mut parts = Vec::new(); + if self.failed > 0 { + parts.push(format!("{} failed", self.failed)); + } + if self.errors > 0 { + parts.push(format!("{} could not be checked", self.errors)); + } + format!( + "{} of {total} check(s): {}", + self.failed + self.errors, + parts.join(", ") + ) + } + + /// One line per check (`PASS`, `FAIL` or `ERROR`, the check, what was observed), the detail of + /// a failure under it, and a summary line. + #[must_use] + pub fn to_text(&self) -> String { + let mut out = String::new(); + for check in &self.checks { + match check.status { + Status::Pass | Status::Fail => { + let word = if check.passed { "PASS" } else { "FAIL" }; + let observed = match &check.observed { + Value::String(text) => text.clone(), + other => other.to_string(), + }; + let _ = writeln!(out, "{word} {} (observed {observed})", check.check); + if let Some(detail) = &check.detail { + for line in detail.lines() { + let _ = writeln!(out, " {line}"); + } + } + } + Status::Error => { + let _ = writeln!( + out, + "ERROR {}: {}", + check.check, + check.detail.as_deref().unwrap_or("could not be checked") + ); + } + } + } + let _ = writeln!(out, "{}", self.summary()); + out + } +} + +/// Reads every check, then runs them all against `analysis`. +/// +/// # Errors +/// +/// Returns a usage error, naming the check, if any check is malformed: nothing is measured then. +/// A check that cannot be measured is not an error here; it is [`Status::Error`] in the result. +pub fn run_all(analysis: &Analysis, checks: &[String]) -> eyre::Result { + let parsed = parse_all(checks)?; + Ok(run_parsed(analysis, &parsed)) +} + +/// Reads every check without running any. +/// +/// # Errors +/// +/// As [`Assertion::parse`], and a usage error if `checks` is empty. +pub fn parse_all(checks: &[String]) -> eyre::Result> { + if checks.is_empty() { + return Err(envelope::usage(format!( + "assert needs at least one --check; the kinds are:\n{}", + kinds_listing() + ))); + } + checks.iter().map(|text| Assertion::parse(text)).collect() +} + +/// Runs checks that were read. +#[must_use] +pub fn run_parsed(analysis: &Analysis, checks: &[Assertion]) -> AssertData { + let checks: Vec = checks.iter().map(|check| check.run(analysis)).collect(); + let failed = checks + .iter() + .filter(|check| check.status == Status::Fail) + .count(); + let errors = checks + .iter() + .filter(|check| check.status == Status::Error) + .count(); + AssertData { + passed: failed == 0 && errors == 0, + failed, + errors, + checks, + } +} + +// ---- findings ------------------------------------------------------------------------------- + +/// `findings kind=K severity=S count<=N`: how many findings (detectors' and an agent's) match. +#[derive(Debug)] +pub struct FindingsCheck { + kind: Option, + severity: Option, + comparison: Comparison, +} + +impl Check for FindingsCheck { + const NAME: &'static str = "findings"; + const ABOUT: &'static str = "findings [kind=K] [severity=error|warn|info] countN (K: log-error, invariant, invariant:, agent-finding, ...)"; + + fn parse(args: &CheckArgs) -> eyre::Result { + args.comparison.require(Self::NAME, &["count"])?; + let pairs = args.pairs(&["kind", "severity"])?; + let severity = pairs + .get("severity") + .map(|text| match text.as_str() { + "error" => Ok(Severity::Error), + "warn" => Ok(Severity::Warn), + "info" => Ok(Severity::Info), + other => Err(envelope::usage(format!( + "severity is error, warn or info, not `{other}`" + ))), + }) + .transpose()?; + Ok(Self { + kind: pairs.get("kind").cloned(), + severity, + comparison: args.comparison.clone(), + }) + } + + fn run(&self, analysis: &Analysis) -> eyre::Result { + let data = analysis.merged_findings(&FindingsQuery { + kind: self.kind.clone(), + severity: self.severity, + limit: None, + })?; + Ok(CheckOutcome::compare( + &self.comparison, + data.total as f64, + || { + let mut lines: Vec = data + .findings + .iter() + .take(5) + .map(|item| { + format!( + "{} {} at {}: {}", + item.id, + item.kind, + timecode::format(item.start_video_s), + item.summary + ) + }) + .collect(); + if data.total > lines.len() { + lines.push(format!("... {} more", data.total - lines.len())); + } + Some(lines.join("\n")) + }, + )) + } +} + +// ---- log ------------------------------------------------------------------------------------ + +/// `log "" count==N`: how many lines of komorebi's log match. +#[derive(Debug)] +pub struct LogCheck { + query: String, + comparison: Comparison, +} + +impl Check for LogCheck { + const NAME: &'static str = "log"; + const ABOUT: &'static str = "log \"\" countN (the query is the `logq` language: level=ERROR and message~\"ghost\")"; + + fn parse(args: &CheckArgs) -> eyre::Result { + args.comparison.require(Self::NAME, &["count"])?; + let query = unwrap_quoted(&args.text); + if query.is_empty() { + return Err(envelope::usage("a `log` check needs a query")); + } + Ok(Self { + query, + comparison: args.comparison.clone(), + }) + } + + fn run(&self, analysis: &Analysis) -> eyre::Result { + let budget = Budget::unlimited().reading_at_most(512 * 1024 * 1024); + let query = |limit: Option, count_only: bool| LogqQuery { + query: self.query.clone(), + from_mach_ns: None, + to_mach_ns: None, + group_by: None, + skip: 0, + limit, + count_only, + }; + + let data = analysis.logq(&query(None, true), &budget)?; + let mut outcome = CheckOutcome::compare(&self.comparison, data.matching as f64, || None); + if !outcome.passed && data.matching > 0 { + let shown = analysis.logq(&query(Some(3), false), &budget)?; + let mut lines: Vec = shown + .hits + .iter() + .map(|hit| { + format!( + "{} {}: {}", + hit.reference, + timecode::format(hit.video_s), + hit.text + ) + }) + .collect(); + if data.matching > lines.len() { + lines.push(format!("... {} more", data.matching - lines.len())); + } + outcome.detail = Some(lines.join("\n")); + } + Ok(outcome) + } +} + +// ---- annotations ---------------------------------------------------------------------------- + +/// `annotations kind=note count>=N`: how many bug regions, notes or findings were recorded. +#[derive(Debug)] +pub struct AnnotationsCheck { + kind: Option, + comparison: Comparison, +} + +impl Check for AnnotationsCheck { + const NAME: &'static str = "annotations"; + const ABOUT: &'static str = "annotations [kind=note|bug|finding] countN"; + + fn parse(args: &CheckArgs) -> eyre::Result { + args.comparison.require(Self::NAME, &["count"])?; + let pairs = args.pairs(&["kind"])?; + let kind = pairs + .get("kind") + .map(|text| match text.as_str() { + "note" => Ok(annotations::Kind::Note), + "bug" => Ok(annotations::Kind::Bug), + "finding" => Ok(annotations::Kind::Finding), + other => Err(envelope::usage(format!( + "annotation kind is note, bug or finding, not `{other}`" + ))), + }) + .transpose()?; + Ok(Self { + kind, + comparison: args.comparison.clone(), + }) + } + + fn run(&self, analysis: &Analysis) -> eyre::Result { + let loaded = annotations::load(&analysis.session.dir)?; + let count = loaded + .annotations + .iter() + .filter(|annotation| self.kind.is_none_or(|kind| annotation.kind == kind)) + .count(); + Ok(CheckOutcome::compare( + &self.comparison, + count as f64, + || None, + )) + } +} + +// ---- video: motion and blank ---------------------------------------------------------------- + +/// The part of a video check that picks what to scan. +#[derive(Debug)] +struct Scan { + crop: Option, + from: Option, + to: Option, +} + +impl Scan { + fn parse(pairs: &BTreeMap) -> eyre::Result { + Ok(Self { + crop: pairs + .get("region") + .map(|text| Crop::parse(text)) + .transpose()?, + from: pairs.get("from").cloned(), + to: pairs.get("to").cloned(), + }) + } + + /// Runs `scan` on every display with video over the range. + fn with( + &self, + analysis: &Analysis, + scan: impl FnOnce(&SessionVideo, &ScanOptions) -> eyre::Result, + ) -> eyre::Result { + let video = SessionVideo::new(&analysis.manifest, analysis.guard()); + let displays = video.displays(); + if displays.is_empty() { + return Err(envelope::unusable("the session recorded no video")); + } + + let (_, end) = analysis.manifest.video_span(); + let moment = |text: &Option, default: f64| -> eyre::Result { + text.as_deref().map_or(Ok(default), |text| { + analysis.resolve(text).map(|resolved| resolved.video_s) + }) + }; + let (from_s, to_s) = (moment(&self.from, 0.0)?, moment(&self.to, end)?); + if to_s < from_s { + return Err(envelope::usage("the end of the range is before its start")); + } + + scan( + &video, + &ScanOptions { + displays: &displays, + from_s, + to_s, + crop: self.crop, + cancel: None, + }, + ) + } +} + +/// `motion region=x,y,w,h from=T to=T segments<=N`: how much the picture moved. +/// +/// The metrics are `segments` (how many stretches of motion), `active_ms` (the time they add up +/// to), `peak_share` (the largest share of the area changing, 0 to 1) and `peak_delta` (the +/// largest mean brightness change, 0 to 255). +#[derive(Debug)] +pub struct MotionCheck { + scan: Scan, + comparison: Comparison, +} + +impl Check for MotionCheck { + const NAME: &'static str = "motion"; + const ABOUT: &'static str = + "motion [region=x,y,w,h] [from=T] [to=T] segments|active_ms|peak_share|peak_deltaN"; + + fn parse(args: &CheckArgs) -> eyre::Result { + args.comparison.require( + Self::NAME, + &["segments", "active_ms", "peak_share", "peak_delta"], + )?; + Ok(Self { + scan: Scan::parse(&args.pairs(&["region", "from", "to"])?)?, + comparison: args.comparison.clone(), + }) + } + + fn run(&self, analysis: &Analysis) -> eyre::Result { + let data = self.scan.with(analysis, |video, options| { + analysis.motion(video, options, 1.0) + })?; + let observed = match self.comparison.metric.as_str() { + "segments" => data.segments.len() as f64, + "active_ms" => (data.active_s() * 1000.0).round(), + "peak_share" => data + .segments + .iter() + .map(|segment| segment.peak_share) + .fold(0.0, f64::max), + _ => data + .segments + .iter() + .map(|segment| segment.peak_mean_delta) + .fold(0.0, f64::max), + }; + Ok(CheckOutcome::compare(&self.comparison, observed, || { + let mut lines: Vec = data + .segments + .iter() + .take(5) + .map(|segment| { + format!( + "display {} {} for {:.0} ms (peak share {:.2})", + segment.display, + timecode::format(segment.start_video_s), + segment.duration_s * 1000.0, + segment.peak_share + ) + }) + .collect(); + if data.segments.len() > lines.len() { + lines.push(format!("... {} more", data.segments.len() - lines.len())); + } + (!lines.is_empty()).then(|| lines.join("\n")) + })) + } +} + +/// `blank region=x,y,w,h from=T to=T color=white ms<=N`: whether a region went one colour. +/// +/// The metrics are `ms` (the longest blank stretch), `total_ms` (all of them added up) and +/// `stretches` (how many). `color` (also spelled `colour`) keeps only stretches of that colour +/// (`white`, `black`, `gray`, or a hex code); without it any blank colour counts. A frozen region +/// is not blank. +#[derive(Debug)] +pub struct BlankCheck { + scan: Scan, + colour: Option, + comparison: Comparison, +} + +impl Check for BlankCheck { + const NAME: &'static str = "blank"; + const ABOUT: &'static str = + "blank region=x,y,w,h [from=T] [to=T] [color=white] ms|total_ms|stretchesN"; + + fn parse(args: &CheckArgs) -> eyre::Result { + args.comparison + .require(Self::NAME, &["ms", "total_ms", "stretches"])?; + let pairs = args.pairs(&["region", "from", "to", "color", "colour"])?; + if !pairs.contains_key("region") { + return Err(envelope::usage("a `blank` check needs region=x,y,w,h")); + } + Ok(Self { + scan: Scan::parse(&pairs)?, + colour: pairs.get("color").or_else(|| pairs.get("colour")).cloned(), + comparison: args.comparison.clone(), + }) + } + + fn run(&self, analysis: &Analysis) -> eyre::Result { + let data = self.scan.with(analysis, |video, options| { + analysis.blank(video, options, DEFAULT_VARIANCE, DEFAULT_MIN_DURATION_S) + })?; + let stretches: Vec<_> = data + .stretches + .iter() + .filter(|stretch| stretch.kind == StretchKind::Blank) + .filter(|stretch| { + self.colour + .as_ref() + .is_none_or(|wanted| stretch.colour.as_ref() == Some(wanted)) + }) + .collect(); + let observed = match self.comparison.metric.as_str() { + "ms" => stretches + .iter() + .map(|stretch| (stretch.duration_s * 1000.0).round()) + .fold(0.0, f64::max), + "total_ms" => stretches + .iter() + .map(|stretch| stretch.duration_s * 1000.0) + .sum::() + .round(), + _ => stretches.len() as f64, + }; + Ok(CheckOutcome::compare(&self.comparison, observed, || { + let mut lines: Vec = stretches + .iter() + .take(5) + .map(|stretch| { + format!( + "display {} blank ({}) at {} for {:.0} ms", + stretch.display, + stretch.colour.as_deref().unwrap_or("?"), + timecode::format(stretch.start_video_s), + stretch.duration_s * 1000.0 + ) + }) + .collect(); + if stretches.len() > lines.len() { + lines.push(format!("... {} more", stretches.len() - lines.len())); + } + (!lines.is_empty()).then(|| lines.join("\n")) + })) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::analysis::envelope::exit_kind; + + fn comparison(text: &str) -> Comparison { + let assertion_text = format!("annotations {text}"); + let captures = TAIL.captures(&assertion_text[12..]).expect("comparison"); + Comparison { + metric: captures[2].to_string(), + op: Op::parse(&captures[3]).expect("op"), + value: captures[4].parse().expect("number"), + } + } + + #[test] + fn every_operator_compares() { + let cases = [ + ("count==2", 2.0, true), + ("count==2", 3.0, false), + ("count!=2", 3.0, true), + ("count!=2", 2.0, false), + ("count<=2", 2.0, true), + ("count<=2", 3.0, false), + ("count>=2", 2.0, true), + ("count>=2", 1.0, false), + ("count<2", 1.0, true), + ("count<2", 2.0, false), + ("count>2", 3.0, true), + ("count>2", 2.0, false), + ]; + for (text, observed, holds) in cases { + assert_eq!( + comparison(text).holds(observed), + holds, + "{text} on {observed}" + ); + } + } + + #[test] + fn spaces_around_the_operator_and_decimals_are_read() { + let tail = TAIL + .captures("region=1,2,3,4 peak_share <= 0.25") + .expect("tail"); + assert_eq!(&tail[1], "region=1,2,3,4"); + assert_eq!(&tail[2], "peak_share"); + assert_eq!(&tail[3], "<="); + assert_eq!(&tail[4], "0.25"); + } + + #[test] + fn words_honour_both_quotes() { + assert_eq!( + words(r#"kind=a "b=c d" 'e f'="g" "#).expect("words"), + ["kind=a", "b=c d", "e f=g"] + ); + assert_eq!( + words("k=\"say \\\"hi\\\"\"").expect("words"), + ["k=say \"hi\""] + ); + assert!(words("k=\"open").is_err()); + } + + #[test] + fn a_log_query_may_be_quoted_or_bare() { + let query = |text: &str| { + let assertion = Assertion::parse(text).expect("parse"); + assertion.text + }; + assert_eq!( + query("log \"level=ERROR\" count==0"), + "log \"level=ERROR\" count==0" + ); + assert_eq!(unwrap_quoted(r#""level=ERROR""#), "level=ERROR"); + assert_eq!(unwrap_quoted("'message~\"ghost\"'"), "message~\"ghost\""); + assert_eq!(unwrap_quoted(r#""a\"b""#), "a\"b"); + // quotes inside a query that does not start with one are kept: they are the query's + assert_eq!( + unwrap_quoted(r#"level=ERROR and message~"ghost""#), + r#"level=ERROR and message~"ghost""# + ); + // a quote that closes before the end does not wrap the whole text + assert_eq!( + unwrap_quoted(r#""a" or message~"b""#), + r#""a" or message~"b""# + ); + } + + #[test] + fn malformed_checks_are_usage_errors_that_list_the_kinds() { + for text in [ + "", + "nonsense count==1", + "findings", + "findings count", + "findings count=1", + "findings count===1", + "findings count<=x", + "findings segments<=1", + "findings kind count<=1", + "findings colour=red count<=1", + "findings severity=loud count<=1", + "findings kind=a kind=b count<=1", + "log count==0", + "motion region=1,2,3 segments==0", + "blank segments==0", + "blank region=1,2,3,4 count==0", + "annotations kind=memo count==0", + ] { + let error = Assertion::parse(text) + .err() + .unwrap_or_else(|| panic!("{text:?}")); + assert_eq!(exit_kind(&error), ExitKind::Usage, "{text:?}: {error}"); + } + + let error = Assertion::parse("nonsense count==1").err().expect("error"); + let message = error.to_string(); + for kind in KINDS { + assert!(message.contains(kind.about), "{message}"); + } + } + + #[test] + fn well_formed_checks_of_every_kind_parse() { + for text in [ + "findings kind=layout-mismatch severity=error count<=0", + "findings count>=1", + "log \"level=ERROR\" count==0", + "log level=ERROR and message~\"ghost\" count<3", + "motion segments<=1", + "motion region=0,0,100,100 from=1.5 to=event:3 active_ms>200", + "motion peak_share<0.2", + "motion peak_delta<=12.5", + "blank region=10,10,300,200 from=bug:a-1 color=white ms<=100", + "blank region=0,0,5,5 total_ms==0", + "annotations kind=note count>=2", + "annotations count!=0", + " findings count<=0 ", + ] { + Assertion::parse(text).unwrap_or_else(|error| panic!("{text:?}: {error}")); + } + } + + #[test] + fn numbers_print_without_noise() { + assert_eq!(number_text(2.0), "2"); + assert_eq!(number_text(0.25), "0.25"); + assert_eq!(number(3.0), serde_json::json!(3)); + assert_eq!(number(0.5), serde_json::json!(0.5)); + assert_eq!(comparison("count<=0").to_string(), "count<=0"); + } + + #[test] + fn errors_are_never_a_pass() { + let data = AssertData { + passed: false, + failed: 0, + errors: 1, + checks: vec![CheckResult { + check: "motion segments<=1".to_string(), + status: Status::Error, + passed: false, + observed: Value::Null, + detail: Some("the session recorded no video".to_string()), + error_kind: Some(ExitKind::Unusable), + }], + }; + assert_eq!(data.exit_kind(), ExitKind::Unusable); + assert!( + data.to_text() + .starts_with("ERROR motion segments<=1: the session recorded no video") + ); + assert!(data.summary().contains("could not be checked")); + } +} diff --git a/komorebi-recorder/src/analysis/envelope.rs b/komorebi-recorder/src/analysis/envelope.rs index aff362ab..c53d4262 100644 --- a/komorebi-recorder/src/analysis/envelope.rs +++ b/komorebi-recorder/src/analysis/envelope.rs @@ -21,6 +21,7 @@ //! | 3 | [`NotFound`](ExitKind::NotFound) | the session, or something asked for in it, does not exist | //! | 4 | [`Unusable`](ExitKind::Unusable) | the session exists but cannot answer: still recording, or it recorded nothing | //! | 5 | [`Budget`](ExitKind::Budget) | the answer would have exceeded the time or size budget | +//! | 6 | [`Assertion`](ExitKind::Assertion) | `assert` measured the recording and a check did not hold | //! //! The codes are stable. [`ENVELOPE_VERSION`] is bumped when the envelope's own fields change; //! the shape of each command's `data` is versioned by its `schema` (`brief/1`), and the @@ -66,7 +67,8 @@ pub const ENVELOPE_VERSION: u32 = 1; /// How the process exits: the kind of a failure, and the exit code that goes with it. /// -/// Serialized in kebab case (`ok`, `error`, `usage`, `not-found`, `unusable`, `budget`) in an +/// Serialized in kebab case (`ok`, `error`, `usage`, `not-found`, `unusable`, `budget`, +/// `assertion`) in an /// envelope's `error.kind`. [`ExitKind::code`] gives the exit code. #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, JsonSchema)] #[serde(rename_all = "kebab-case")] @@ -86,10 +88,13 @@ pub enum ExitKind { /// The answer would have exceeded the time or size budget: exit code 5. Narrowing the request /// is the way forward. Budget, + /// `assert` measured the recording and a check did not hold: exit code 6. Not a failure of the + /// command: the answer is "no". + Assertion, } impl ExitKind { - /// Returns the process exit code of this kind: 0 for [`ExitKind::Ok`], then 1 to 5 in the order + /// Returns the process exit code of this kind: 0 for [`ExitKind::Ok`], then 1 to 6 in the order /// the variants are declared. /// /// The mapping is stable, so a script may rely on the numbers (`exit_codes_are_stable` pins @@ -104,6 +109,7 @@ impl ExitKind { /// assert_eq!(ExitKind::Usage.code(), 2); /// assert_eq!(ExitKind::NotFound.code(), 3); /// assert_eq!(ExitKind::Budget.code(), 5); + /// assert_eq!(ExitKind::Assertion.code(), 6); /// ``` #[must_use] pub fn code(self) -> u8 { @@ -114,6 +120,7 @@ impl ExitKind { Self::NotFound => 3, Self::Unusable => 4, Self::Budget => 5, + Self::Assertion => 6, } } } @@ -248,6 +255,32 @@ pub fn exit_kind(report: &eyre::Report) -> ExitKind { .map_or(ExitKind::Error, |error| error.kind) } +/// Marks a failure whose answer the command has already printed, so that +/// `crate::cli::execute` only sets the exit code and prints nothing more (no second envelope). +/// +/// Made by [`reported`]; the kind is still read with [`exit_kind`]. +#[derive(Debug)] +pub struct Reported; + +impl fmt::Display for Reported { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("the answer was already printed") + } +} + +impl std::error::Error for Reported {} + +/// Returns an error of this kind whose answer the command has already printed (see [`Reported`]). +pub fn reported(kind: ExitKind, message: impl Into) -> eyre::Report { + failure(kind, message).wrap_err(Reported) +} + +/// Returns whether `report` was made by [`reported`]. +#[must_use] +pub fn is_reported(report: &eyre::Report) -> bool { + report.downcast_ref::().is_some() +} + /// The `error` of a failed [`Envelope`]: the kind of failure and what to do about it. #[derive(Debug, Serialize, JsonSchema)] pub struct ErrorBody { @@ -332,6 +365,21 @@ impl Envelope { } } + /// Returns the envelope as a failed one that still carries its `data`: `ok` is `false` and + /// `error` has `kind` and its exit code. For a command whose answer is the point even when it + /// is "no" (`assert`). + #[must_use] + pub fn with_error(mut self, kind: ExitKind, message: impl Into) -> Self { + self.ok = false; + self.error = Some(ErrorBody { + kind, + code: kind.code(), + message: message.into(), + hint: None, + }); + self + } + /// Returns the envelope with `warnings` as its warnings. They replace any that were there. #[must_use] pub fn warnings(mut self, warnings: Vec) -> Self { @@ -415,11 +463,12 @@ mod tests { ExitKind::NotFound, ExitKind::Unusable, ExitKind::Budget, + ExitKind::Assertion, ] .iter() .map(|kind| kind.code()) .collect(); - assert_eq!(codes, [0, 1, 2, 3, 4, 5]); + assert_eq!(codes, [0, 1, 2, 3, 4, 5, 6]); } #[test] diff --git a/komorebi-recorder/src/analysis/mod.rs b/komorebi-recorder/src/analysis/mod.rs index 65a81757..76272ccf 100644 --- a/komorebi-recorder/src/analysis/mod.rs +++ b/komorebi-recorder/src/analysis/mod.rs @@ -8,6 +8,7 @@ //! bound ([`budget`]). pub mod answers; +pub mod assertions; pub mod blank; pub mod blocks; pub mod brief; diff --git a/komorebi-recorder/src/analysis/schemas.rs b/komorebi-recorder/src/analysis/schemas.rs index c82631f8..e6738ddf 100644 --- a/komorebi-recorder/src/analysis/schemas.rs +++ b/komorebi-recorder/src/analysis/schemas.rs @@ -174,6 +174,11 @@ pub const ALL: &[Entry] = &[ about: "`findings`: detector findings (with stable ids) and an agent's findings, together", schema: of::, }, + Entry { + name: "assert/1", + about: "`assert`: pass, fail or error for each scripted check, with what was observed", + schema: of::, + }, Entry { name: "record/1", about: "`record --json`: the finished session, its displays and frames, warnings and the --until-exit command's status", diff --git a/komorebi-recorder/src/cli/assert.rs b/komorebi-recorder/src/cli/assert.rs new file mode 100644 index 00000000..da0b29fe --- /dev/null +++ b/komorebi-recorder/src/cli/assert.rs @@ -0,0 +1,63 @@ +//! `assert`: scripted pass/fail checks over a recording, for an agent's own scenario tests. +//! +//! Each `--check` is a kind, its arguments and a comparison (see +//! [`crate::analysis::assertions`] for the grammar and the kinds). The answer is a line per check +//! and a summary; with `--json` it is the `assert/1` envelope. The exit code is the result: 0 when +//! every check passes, 6 when one does not hold, and 2, 3 or 4 when the checks are malformed or +//! cannot be measured. Read-only: it never writes to the session. +//! +//! Covered by `tests/assert.rs`, which runs the command through the real binary. + +use super::print_json; +use crate::analysis::Analysis; +use crate::analysis::assertions; +use crate::analysis::assertions::SCHEMA_ASSERT; +use crate::analysis::envelope::Envelope; +use crate::analysis::envelope::ExitKind; +use crate::analysis::envelope::reported; +use clap::Args; +use color_eyre::eyre; +use std::path::PathBuf; + +/// The arguments of `assert`. +#[derive(Args, Debug)] +pub struct AssertArgs { + /// Session directory + pub session: PathBuf, + /// A check, ` ` (repeatable): `findings kind=layout-mismatch + /// count<=0`, `log "level=ERROR" count==0`, `motion region=0,0,800,600 segments<=2`, + /// `blank region=0,0,800,600 color=white ms<=100`, `annotations kind=note count>=1`. Operators: + /// == != <= >= < >. Exits 0 if all pass, 6 if one fails + #[arg(long = "check", value_name = "CHECK", required = true)] + pub checks: Vec, + /// Machine-readable JSON output + #[arg(long)] + pub json: bool, +} + +pub(super) fn assert(args: &AssertArgs) -> eyre::Result<()> { + // read every check before opening the session: a typo is a usage error whatever the session is + let parsed = assertions::parse_all(&args.checks)?; + let analysis = Analysis::open(&args.session)?; + let data = assertions::run_parsed(&analysis, &parsed); + + let kind = data.exit_kind(); + let summary = data.summary(); + if args.json { + let mut envelope = Envelope::ok(SCHEMA_ASSERT, data).warnings(analysis.warnings()); + if kind != ExitKind::Ok { + envelope = envelope.with_error(kind, summary.clone()); + } + print_json(&envelope)?; + } else { + for warning in analysis.warnings() { + eprintln!("note: {warning}"); + } + print!("{}", data.to_text()); + } + + match kind { + ExitKind::Ok => Ok(()), + kind => Err(reported(kind, summary)), + } +} diff --git a/komorebi-recorder/src/cli/mod.rs b/komorebi-recorder/src/cli/mod.rs index 6073ba09..7a57784f 100644 --- a/komorebi-recorder/src/cli/mod.rs +++ b/komorebi-recorder/src/cli/mod.rs @@ -7,6 +7,7 @@ mod agent; mod annotate; +mod assert; mod doctor; mod findings; mod investigate; @@ -38,6 +39,7 @@ pub use agent::SchemaArgs; pub use agent::SearchArgs; pub use annotate::AnnotateArgs; pub use annotate::AnnotationsArgs; +pub use assert::AssertArgs; pub use doctor::DoctorArgs; pub use findings::FindingsArgs; pub use investigate::CodeArgs; @@ -148,6 +150,8 @@ pub enum Command { Annotate(AnnotateArgs), /// List a session's bug regions, notes and findings Annotations(AnnotationsArgs), + /// Pass/fail checks over a recording (findings, log, motion, blank, notes); exit 6 if one fails + Assert(AssertArgs), /// Drop a named marker onto the timeline of a recording that is still going Mark(mark::MarkArgs), /// Rebuild a session's timeline from its raw files, recovering it first if the recorder died @@ -226,6 +230,7 @@ impl Command { Self::Annotate(args) => args.json(), Self::Annotations(args) => args.json, Self::Mark(args) => args.json, + Self::Assert(args) => args.json, Self::Doctor(args) => args.json, Self::Record(args) => args.json, Self::Displays(args) => args.json, @@ -248,6 +253,10 @@ pub fn execute(command: Command) -> std::process::ExitCode { Ok(()) => std::process::ExitCode::SUCCESS, Err(error) => { let kind = crate::analysis::envelope::exit_kind(&error); + if crate::analysis::envelope::is_reported(&error) { + // the command printed its own answer; only the exit status is left to set + return std::process::ExitCode::from(kind.code()); + } if json { let envelope = crate::analysis::envelope::Envelope::failed(&error); if let Ok(text) = serde_json::to_string_pretty(&envelope) { @@ -299,6 +308,7 @@ pub fn run(command: Command) -> eyre::Result<()> { Command::Annotate(args) => annotate::annotate(&args), Command::Annotations(args) => annotate::list(&args), Command::Mark(args) => mark::mark(&args), + Command::Assert(args) => assert::assert(&args), Command::Reindex(args) => reindex(&args), Command::Selftest(args) => crate::selftest::run(&args), Command::SelftestClock => crate::selftest::run_clock(), diff --git a/komorebi-recorder/src/guide.md b/komorebi-recorder/src/guide.md index 893c6b03..15ef7806 100644 --- a/komorebi-recorder/src/guide.md +++ b/komorebi-recorder/src/guide.md @@ -10,7 +10,7 @@ what *produces* the bug the user described, from the recording, and then propose what it shows. Every command takes `--json`; its output is an envelope `{ok, schema, data, warnings, truncated, next_cursor, error}` whose `data` is described by `komorebi-recorder schema `. Exit codes: 0 ok, 1 error, 2 usage, 3 not found, 4 unusable -session, 5 over budget. +session, 5 over budget, 6 an `assert` check failed. ## 1. Start with the brief @@ -130,6 +130,18 @@ the command prints the line it resolved to, so check it is the statement you mea text lists the candidates and is refused. Never claim a cause without a reference, and say what you could not verify. +### Scripted checks + +To turn a scenario into pass or fail, `komorebi-recorder assert --check "" ...` evaluates checks with +the same analyses and answers with its exit code (0 all pass, 6 one failed, 4 one could not be measured, 2 malformed). +A check is ` `, with `== != <= >= < >`: + + findings kind=layout-mismatch count<=0 + log "level=ERROR and message~\"ghost\"" count==0 (the logq language) + motion region=0,0,800,600 from=bug:a-1.start segments<=2 (also active_ms, peak_share) + blank region=0,0,800,600 color=white ms<=100 + annotations kind=note count>=3 (did the script's `mark`s land?) + ## 5. Propose the fix Name the file and function from `code`. Explain the chain from what the user saw to the state to the diff --git a/komorebi-recorder/tests/assert.rs b/komorebi-recorder/tests/assert.rs new file mode 100644 index 00000000..b3258356 --- /dev/null +++ b/komorebi-recorder/tests/assert.rs @@ -0,0 +1,349 @@ +//! `assert` through the real binary: checks over findings, the log, annotations, motion and blank +//! regions, and what the exit code says. + +use komorebi_recorder::testkit; +use komorebi_recorder::testkit::SessionBuilder; +use serde_json::Value; +use std::process::Command; +use std::process::Output; + +fn recorder(args: &[&str]) -> Output { + Command::new(env!("CARGO_BIN_EXE_komorebi-recorder")) + .args(args) + .output() + .expect("run komorebi-recorder") +} + +fn stdout(output: &Output) -> String { + String::from_utf8_lossy(&output.stdout).into_owned() +} + +fn stderr(output: &Output) -> String { + String::from_utf8_lossy(&output.stderr).into_owned() +} + +/// A session with an error and a warning in komorebi's log, and one note. +fn session(name: &str) -> String { + let (session, _) = SessionBuilder::new(name) + .log(2.0, "INFO", "komorebi::process_event: processing event") + .log( + 3.0, + "ERROR", + "komorebi::window: could not place window 4711", + ) + .log(4.0, "WARN", "komorebi::reaper: window 12 is gone") + .build(); + let dir = session.dir.display().to_string(); + + let added = recorder(&[ + "annotate", &dir, "add", "--kind", "note", "--start", "2.5", "--text", "step one", + ]); + assert!(added.status.success(), "{}", stderr(&added)); + dir +} + +#[test] +fn passing_checks_exit_zero_and_print_a_line_each() { + let dir = session("assert-pass"); + let output = recorder(&[ + "assert", + &dir, + "--check", + "log \"level=ERROR\" count==1", + "--check", + "log message~\"window 12\" count>=1", + "--check", + "findings kind=layout-mismatch count<=0", + "--check", + "annotations kind=note count>=1", + "--check", + "annotations kind=bug count==0", + ]); + assert_eq!(output.status.code(), Some(0), "{}", stderr(&output)); + + let text = stdout(&output); + let lines: Vec<&str> = text.lines().collect(); + assert_eq!(lines.len(), 6, "{text}"); + assert_eq!(lines[0], "PASS log \"level=ERROR\" count==1 (observed 1)"); + assert_eq!( + lines[2], + "PASS findings kind=layout-mismatch count<=0 (observed 0)" + ); + assert_eq!(lines[5], "all 5 check(s) passed"); +} + +#[test] +fn a_failing_check_exits_six_and_is_named() { + let dir = session("assert-fail"); + let output = recorder(&[ + "assert", + &dir, + "--check", + "log \"level=ERROR\" count==0", + "--check", + "findings kind=log-error count==1", + "--check", + "annotations kind=note count>=3", + ]); + assert_eq!(output.status.code(), Some(6), "{}", stderr(&output)); + + let text = stdout(&output); + assert!( + text.contains("FAIL log \"level=ERROR\" count==0 (observed 1)"), + "{text}" + ); + // the line that broke it is shown under the failure + assert!(text.contains("could not place window 4711"), "{text}"); + assert!( + text.contains("PASS findings kind=log-error count==1 (observed 1)"), + "{text}" + ); + assert!( + text.contains("FAIL annotations kind=note count>=3 (observed 1)"), + "{text}" + ); + assert!(text.contains("2 of 3 check(s): 2 failed"), "{text}"); +} + +#[test] +fn json_is_an_assert_envelope_and_the_exit_code_still_says_the_result() { + let dir = session("assert-json"); + + let passing = recorder(&[ + "assert", + &dir, + "--check", + "log \"level=WARN\" count==1", + "--json", + ]); + assert_eq!(passing.status.code(), Some(0)); + let envelope: Value = serde_json::from_slice(&passing.stdout).expect("JSON"); + assert_eq!(envelope["ok"], true); + assert_eq!(envelope["schema"], "assert/1"); + assert_eq!(envelope["data"]["passed"], true); + assert_eq!(envelope["data"]["checks"][0]["passed"], true); + assert_eq!(envelope["data"]["checks"][0]["observed"], 1); + assert_eq!( + envelope["data"]["checks"][0]["check"], + "log \"level=WARN\" count==1" + ); + + let failing = recorder(&[ + "assert", + &dir, + "--check", + "log \"level=ERROR\" count==0", + "--check", + "log \"level=INFO\" count==1", + "--json", + ]); + assert_eq!(failing.status.code(), Some(6)); + // one document, not an answer followed by an error envelope + let envelope: Value = serde_json::from_slice(&failing.stdout).expect("one JSON document"); + assert_eq!(envelope["ok"], false); + assert_eq!(envelope["schema"], "assert/1"); + assert_eq!(envelope["error"]["kind"], "assertion"); + assert_eq!(envelope["error"]["code"], 6); + assert_eq!(envelope["data"]["passed"], false); + assert_eq!(envelope["data"]["failed"], 1); + let checks = envelope["data"]["checks"].as_array().expect("checks"); + assert_eq!(checks[0]["passed"], false); + assert_eq!(checks[0]["status"], "fail"); + assert_eq!(checks[0]["observed"], 1); + assert!( + checks[0]["detail"] + .as_str() + .expect("detail") + .contains("4711") + ); + assert_eq!(checks[1]["passed"], true); + + let schema = recorder(&["schema", "assert/1"]); + assert!(schema.status.success(), "{}", stderr(&schema)); +} + +#[test] +fn malformed_checks_are_usage_errors_that_list_the_kinds() { + let dir = session("assert-usage"); + + for bad in [ + "nonsense count==1", + "findings", + "findings count=1", + "findings count<=lots", + "log count==0", + "motion bogus=1 segments==0", + "blank segments==0", + ] { + let output = recorder(&["assert", &dir, "--check", bad]); + assert_eq!(output.status.code(), Some(2), "{bad}: {}", stderr(&output)); + } + + let output = recorder(&["assert", &dir, "--check", "nonsense count==1"]); + let message = stderr(&output); + for kind in ["findings", "log", "motion", "blank", "annotations"] { + assert!(message.contains(kind), "{message}"); + } + + // a bad logq query is a usage error of that check, not a pass; so is no check at all + let output = recorder(&["assert", &dir, "--check", "log \"level=\" count==0"]); + assert_eq!(output.status.code(), Some(2), "{}", stderr(&output)); + assert_eq!(recorder(&["assert", &dir]).status.code(), Some(2)); + + // a typo in one check measures nothing, whatever the others would say + let output = recorder(&[ + "assert", + &dir, + "--check", + "log \"level=ERROR\" count==1", + "--check", + "oops count==0", + ]); + assert_eq!(output.status.code(), Some(2)); + assert_eq!(stdout(&output), ""); +} + +#[test] +fn a_session_that_does_not_exist_is_not_found() { + let output = recorder(&[ + "assert", + "no-such-session-dir", + "--check", + "findings count==0", + ]); + assert_eq!(output.status.code(), Some(3), "{}", stderr(&output)); +} + +#[test] +fn a_check_that_cannot_be_measured_is_an_error_not_a_pass() { + // no video was recorded, so a motion check has nothing to look at + let dir = session("assert-no-video"); + let output = recorder(&[ + "assert", + &dir, + "--check", + "annotations count>=1", + "--check", + "motion segments<=100", + "--json", + ]); + assert_eq!(output.status.code(), Some(4), "{}", stderr(&output)); + let envelope: Value = serde_json::from_slice(&output.stdout).expect("JSON"); + assert_eq!(envelope["ok"], false); + assert_eq!(envelope["error"]["kind"], "unusable"); + assert_eq!(envelope["data"]["passed"], false); + assert_eq!(envelope["data"]["errors"], 1); + assert_eq!(envelope["data"]["checks"][0]["status"], "pass"); + let motion = &envelope["data"]["checks"][1]; + assert_eq!(motion["status"], "error"); + assert_eq!(motion["passed"], false); + assert!( + motion["detail"] + .as_str() + .expect("detail") + .contains("no video") + ); + + let text = recorder(&["assert", &dir, "--check", "motion segments<=100"]); + assert_eq!(text.status.code(), Some(4)); + assert!( + stdout(&text).starts_with("ERROR motion segments<=100"), + "{}", + stdout(&text) + ); +} + +fn walk(dir: &std::path::Path) -> Vec<(String, u64)> { + let mut out = Vec::new(); + for entry in std::fs::read_dir(dir).expect("read dir").flatten() { + let path = entry.path(); + if path.is_dir() { + out.extend(walk(&path)); + } else { + out.push(( + path.display().to_string(), + entry.metadata().map_or(0, |meta| meta.len()), + )); + } + } + out.sort(); + out +} + +#[test] +fn assert_leaves_the_session_as_it_found_it() { + let dir = session("assert-readonly"); + let before = walk(std::path::Path::new(&dir)); + let _ = recorder(&[ + "assert", + &dir, + "--check", + "findings count>=0", + "--check", + "log \"level=ERROR\" count==1", + ]); + assert_eq!(walk(std::path::Path::new(&dir)), before); +} + +#[test] +fn motion_and_blank_checks_measure_the_video() { + if !testkit::ffmpeg_available() { + eprintln!("skipped: ffmpeg is not available to make a test video"); + return; + } + + // 3 s at 30 fps: counting for a second, still for half a second, white for half a second, + // counting again + let (session, _) = SessionBuilder::new("assert-video") + .video_showing(1, 0.0, 90, 30, |frame| match frame { + 0..=29 => frame, + 30..=44 => 29, + 45..=59 => 0xFFFF, + _ => frame, + }) + .build(); + let dir = session.dir.display().to_string(); + + let output = recorder(&[ + "assert", + &dir, + "--check", + "motion region=500,0,500,800 segments>=2", + "--check", + "motion region=0,0,500,800 to=1.4 segments==0", + "--check", + "motion from=1.1 to=1.4 active_ms<=100", + "--check", + "blank region=0,0,1000,800 color=white ms>=300", + "--check", + "blank region=0,0,1000,800 color=white ms<=100", + "--check", + "blank region=0,0,1000,800 from=0.5 color=black stretches==0", + ]); + let text = stdout(&output); + assert_eq!(output.status.code(), Some(6), "{text}{}", stderr(&output)); + let lines: Vec<&str> = text.lines().collect(); + assert!( + lines[0].starts_with("PASS motion region=500,0,500,800 segments>=2"), + "{text}" + ); + assert!( + lines[1].starts_with("PASS motion region=0,0,500,800 to=1.4 segments==0"), + "{text}" + ); + assert!(lines[2].starts_with("PASS motion from=1.1"), "{text}"); + assert!(lines[3].starts_with("PASS blank"), "{text}"); + assert!(lines[4].starts_with("FAIL blank"), "{text}"); + assert!(lines[6].starts_with("PASS blank"), "{text}"); + + let answer: Value = serde_json::from_slice( + &recorder(&["assert", &dir, "--check", "motion peak_share>0", "--json"]).stdout, + ) + .expect("JSON"); + assert_eq!(answer["data"]["passed"], true); + assert!(answer["data"]["checks"][0]["observed"].is_number()); + + // times are TimeSpecs; one that does not exist is that check's error + let output = recorder(&["assert", &dir, "--check", "motion from=event:9 segments==0"]); + assert_eq!(output.status.code(), Some(3), "{}", stdout(&output)); +}