diff --git a/nix/tests/vm/lib/komorebi_vm/status.py b/nix/tests/vm/lib/komorebi_vm/status.py new file mode 100644 index 00000000..61e1ff8c --- /dev/null +++ b/nix/tests/vm/lib/komorebi_vm/status.py @@ -0,0 +1,167 @@ +"""The scenario-by-port status table, from the runs themselves. + +KDE prints `== [scope] module.function` before the checks of each step of a suite, and each check +as ` ok [scope] ...`, ` FAIL ...`, ` skip ...` or ` known failure ...`; the log of a +`nix build -L` has a prefix on every line, which is dropped. kfw writes a `result.json` for each +scenario, with the passed, failed, skipped and known checks, each with its scope. This reads both +and makes one row for each scenario (the module name: `rules_exe`, `commands`) and one column for +each port, so that the table in docs/dev/BEHAVIOUR.md says what the last run said and not what +somebody wrote. Only the checks labelled `shared` are the scenario's; what a platform labelled +with its own name is an extra, and what a scenario needed to start has no label and is left out. + +Only the standard library: `scripts/scenario-status` loads this file by its path. +""" + +import json +import re +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + +PREFIX = re.compile(r"^[\w.-]+> ") +STEP = re.compile(r"^== \[(?P[\w-]+)\] (?P[\w.]+)(?P: left out by \w+)?\s*$") +RESULT = re.compile(r"^ (?Pok |FAIL|skip|known failure) ") + + +@dataclass +class Counts: + passed: int = 0 + failed: int = 0 + skipped: int = 0 + known: int = 0 + + def add(self, other: "Counts") -> None: + self.passed += other.passed + self.failed += other.failed + self.skipped += other.skipped + self.known += other.known + + def cell(self) -> str: + """`pass 9`, `FAIL 2 of 9`, with `, 1 known` and `, 1 skipped` after.""" + total = self.passed + self.failed + head = f"pass {self.passed}" if not self.failed else f"FAIL {self.failed} of {total}" + if not total and not (self.known or self.skipped): + head = "-" + parts = [head] + if self.known: + parts.append(f"{self.known} known") + if self.skipped: + parts.append(f"{self.skipped} skipped") + return ", ".join(parts) + + +@dataclass +class Status: + """The counts of each scenario, for each port.""" + + shared: dict[str, dict[str, Counts]] = field(default_factory=dict) + """scenario -> port -> counts, for the steps labelled `shared` (and every kfw scenario).""" + extras: dict[str, dict[str, Counts]] = field(default_factory=dict) + """extra -> port -> counts, for the steps a platform labelled with its own name.""" + + def _counts(self, table: dict[str, dict[str, Counts]], name: str, port: str) -> Counts: + return table.setdefault(name, {}).setdefault(port, Counts()) + + +def read_kde(status: Status, text: str) -> None: + """Adds the checks of a log of a run on KDE.""" + scenario: str | None = None + table = status.shared + for raw in text.splitlines(): + line = PREFIX.sub("", raw) + if step := STEP.match(line): + if step["left_out"]: + scenario = None + continue + module = step["name"].split(".")[0] + scenario = module + table = status.shared if step["scope"] == "shared" else status.extras + continue + if scenario is None: + continue + if result := RESULT.match(line): + counts = status._counts(table, scenario, "kde") + kind = result["kind"].strip() + if kind == "ok": + counts.passed += 1 + elif kind == "FAIL": + counts.failed += 1 + elif kind == "skip": + counts.skipped += 1 + else: + counts.known += 1 + + +def read_windows(status: Status, directory: Path) -> None: + """Adds the latest `result.json` of each scenario under `directory`, by modification time.""" + latest: dict[str, tuple[float, dict[str, Any]]] = {} + for path in directory.rglob("result.json"): + try: + data = json.loads(path.read_text()) + except (OSError, ValueError): + continue + name = data.get("scenario") + if isinstance(name, str) and path.stat().st_mtime >= latest.get(name, (0.0, {}))[0]: + latest[name] = (path.stat().st_mtime, data) + for name, (_, data) in latest.items(): + checks = [c for c in data.get("checks", []) if isinstance(c, dict)] + if not any(c.get("scope") for c in checks): + # a result with no scopes: the totals are all there is + counts = status._counts(status.shared, name, "windows") + counts.passed = int(data.get("passed", 0)) + counts.failed = int(data.get("failed", 0)) + counts.skipped = int(data.get("skipped", 0)) + counts.known = int(data.get("known_failures", 0)) + continue + for check in checks: + scope = check.get("scope") + if not scope: + continue # what a scenario needed to start, not what it checks + if scope == "shared" and not name.startswith("windows_"): + counts = status._counts(status.shared, name, "windows") + elif scope == "shared": + continue # the shared steps an extra's run starts from belong to their scenarios + else: + counts = status._counts(status.extras, name, "windows") + if check.get("skipped"): + counts.skipped += 1 + elif check.get("known_failure"): + counts.known += 1 + elif check.get("passed"): + counts.passed += 1 + else: + counts.failed += 1 + + +def _cell(counts: Counts) -> str: + return counts.cell() if counts.passed + counts.failed + counts.known + counts.skipped else "-" + + +def table(status: Status) -> str: + """The Markdown tables: the shared scenarios on each port, then the extras of each.""" + ports = ("kde", "windows") + out = ["| scenario | kde | windows |", "|---|---|---|"] + for name in sorted(status.shared): + cells = [status.shared[name].get(port, Counts()) for port in ports] + out.append( + f"| `{name}` | " + + " | ".join( + c.cell() if c.passed + c.failed + c.known + c.skipped else "not run" for c in cells + ) + + " |" + ) + out += ["", "Only on one port:", "", "| extra | kde | windows |", "|---|---|---|"] + for name in sorted(status.extras): + cells = [status.extras[name].get(port, Counts()) for port in ports] + out.append(f"| `{name}` | " + " | ".join(_cell(c) for c in cells) + " |") + return "\n".join(out) + "\n" + + +BEGIN = "" +END = "" + + +def replace_region(document: str, content: str) -> str: + """The document with what is between the two markers replaced by `content`.""" + start, end = document.index(BEGIN) + len(BEGIN), document.index(END) + return f"{document[:start]}\n{content}{document[end:]}" diff --git a/nix/tests/vm/lib/tests/test_status.py b/nix/tests/vm/lib/tests/test_status.py new file mode 100644 index 00000000..fc2f5443 --- /dev/null +++ b/nix/tests/vm/lib/tests/test_status.py @@ -0,0 +1,67 @@ +from pathlib import Path + +from komorebi_vm import status + +KDE_LOG = """\ +vm-test-run-komorebi-x> == [kde] core_e2e.start +vm-test-run-komorebi-x> ok [kde] the daemon loaded its script +vm-test-run-komorebi-x> == [shared] tiling.run +vm-test-run-komorebi-x> ok [shared] every window is where the layout says +vm-test-run-komorebi-x> FAIL [shared] the windows share the work area +vm-test-run-komorebi-x> [shared] tiling.run: 3.2 s +vm-test-run-komorebi-x> == [shared] focus.run: left out by KFDE_ONLY +vm-test-run-komorebi-x> ok [shared] not counted: the step was left out +vm-test-run-komorebi-x> == [shared] commands.unknown_monitor +vm-test-run-komorebi-x> known failure [shared] a window is not lost: the container is removed +vm-test-run-komorebi-x> skip [shared] change-layout x: the port has no such layout +""" + + +def test_kde_counts_are_by_scenario_and_scope() -> None: + found = status.Status() + status.read_kde(found, KDE_LOG) + tiling = found.shared["tiling"]["kde"] + assert (tiling.passed, tiling.failed) == (1, 1) + assert "focus" not in found.shared + unknown = found.shared["commands"]["kde"] + assert (unknown.known, unknown.skipped) == (1, 1) + assert found.extras["core_e2e"]["kde"].passed == 1 + + +def test_windows_counts_only_the_shared_checks_of_a_scenario(tmp_path: Path) -> None: + (tmp_path / "a").mkdir() + (tmp_path / "a" / "result.json").write_text( + '{"scenario": "closing", "checks": [' + '{"check": "setup", "passed": true},' + '{"check": "one", "passed": true, "scope": "shared"},' + '{"check": "two", "passed": false, "scope": "shared"},' + '{"check": "three", "passed": true, "scope": "windows"}]}' + ) + found = status.Status() + status.read_windows(found, tmp_path) + closing = found.shared["closing"]["windows"] + assert (closing.passed, closing.failed) == (1, 1) + assert found.extras["closing"]["windows"].passed == 1 + + +def test_a_windows_result_without_scopes_is_its_totals(tmp_path: Path) -> None: + (tmp_path / "result.json").write_text( + '{"scenario": "tiling", "passed": 10, "failed": 0, "skipped": 0, "known_failures": 0}' + ) + found = status.Status() + status.read_windows(found, tmp_path) + assert found.shared["tiling"]["windows"].passed == 10 + + +def test_the_table_has_a_row_for_each_scenario_and_a_dash_for_a_port_that_did_not_run() -> None: + found = status.Status() + status.read_kde(found, KDE_LOG) + out = status.table(found) + assert "| `tiling` | FAIL 1 of 2 | not run |" in out or "| `tiling` | FAIL 1 of 2 | - |" in out + + +def test_the_region_between_the_markers_is_replaced() -> None: + document = f"before\n{status.BEGIN}\nold\n{status.END}\nafter\n" + assert status.replace_region(document, "new\n") == ( + f"before\n{status.BEGIN}\nnew\n{status.END}\nafter\n" + ) diff --git a/scripts/scenario-status b/scripts/scenario-status new file mode 100755 index 00000000..95693b19 --- /dev/null +++ b/scripts/scenario-status @@ -0,0 +1,36 @@ +#!/usr/bin/env python3 +"""Makes the scenario-by-port status table from the logs of runs on KDE and the result files of +kfw (see nix/tests/vm/lib/komorebi_vm/status.py). + + scripts/scenario-status --kde LOG... [--windows DIR...] [--update docs/dev/BEHAVIOUR.md] + +Without --update the table is printed. With it, what is between the two comment markers +(`` and ``) of the file is replaced. +""" + +import argparse +import importlib.util +from pathlib import Path + +path = Path(__file__).resolve().parent.parent / "nix/tests/vm/lib/komorebi_vm/status.py" +spec = importlib.util.spec_from_file_location("status", path) +assert spec and spec.loader +status = importlib.util.module_from_spec(spec) +spec.loader.exec_module(status) + +parser = argparse.ArgumentParser(description=__doc__.split("\n")[0]) +parser.add_argument("--kde", nargs="*", default=[], type=Path, help="logs of runs on KDE") +parser.add_argument("--windows", nargs="*", default=[], type=Path, help="directories of kfw results") +parser.add_argument("--update", type=Path, help="the document to put the table in") +args = parser.parse_args() + +found = status.Status() +for log in args.kde: + status.read_kde(found, log.read_text(errors="replace")) +for directory in args.windows: + status.read_windows(found, directory) +text = status.table(found) +if args.update: + args.update.write_text(status.replace_region(args.update.read_text(), text)) +else: + print(text, end="")