feat(repo): scenario-status makes the table from the runs

This commit is contained in:
LGUG2Z committed 2026-10-30 12:03:01 -07:00
1 parent 781b082778
commit 37bb9e3fe4
3 files changed
+270

No files matched your search

+167
View File
@@ -0,0 +1,167 @@
"""The scenario-by-port status table, from the runs themselves.
KDE prints `== [scope] module.function` before the checks of each step of a suite, and each check
as ` ok [scope] ...`, ` FAIL ...`, ` skip ...` or ` known failure ...`; the log of a
`nix build -L` has a prefix on every line, which is dropped. kfw writes a `result.json` for each
scenario, with the passed, failed, skipped and known checks, each with its scope. This reads both
and makes one row for each scenario (the module name: `rules_exe`, `commands`) and one column for
each port, so that the table in docs/dev/BEHAVIOUR.md says what the last run said and not what
somebody wrote. Only the checks labelled `shared` are the scenario's; what a platform labelled
with its own name is an extra, and what a scenario needed to start has no label and is left out.
Only the standard library: `scripts/scenario-status` loads this file by its path.
"""
import json
import re
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any
PREFIX = re.compile(r"^[\w.-]+> ")
STEP = re.compile(r"^== \[(?P<scope>[\w-]+)\] (?P<name>[\w.]+)(?P<left_out>: left out by \w+)?\s*$")
RESULT = re.compile(r"^ (?P<kind>ok |FAIL|skip|known failure) ")
@dataclass
class Counts:
passed: int = 0
failed: int = 0
skipped: int = 0
known: int = 0
def add(self, other: "Counts") -> None:
self.passed += other.passed
self.failed += other.failed
self.skipped += other.skipped
self.known += other.known
def cell(self) -> str:
"""`pass 9`, `FAIL 2 of 9`, with `, 1 known` and `, 1 skipped` after."""
total = self.passed + self.failed
head = f"pass {self.passed}" if not self.failed else f"FAIL {self.failed} of {total}"
if not total and not (self.known or self.skipped):
head = "-"
parts = [head]
if self.known:
parts.append(f"{self.known} known")
if self.skipped:
parts.append(f"{self.skipped} skipped")
return ", ".join(parts)
@dataclass
class Status:
"""The counts of each scenario, for each port."""
shared: dict[str, dict[str, Counts]] = field(default_factory=dict)
"""scenario -> port -> counts, for the steps labelled `shared` (and every kfw scenario)."""
extras: dict[str, dict[str, Counts]] = field(default_factory=dict)
"""extra -> port -> counts, for the steps a platform labelled with its own name."""
def _counts(self, table: dict[str, dict[str, Counts]], name: str, port: str) -> Counts:
return table.setdefault(name, {}).setdefault(port, Counts())
def read_kde(status: Status, text: str) -> None:
"""Adds the checks of a log of a run on KDE."""
scenario: str | None = None
table = status.shared
for raw in text.splitlines():
line = PREFIX.sub("", raw)
if step := STEP.match(line):
if step["left_out"]:
scenario = None
continue
module = step["name"].split(".")[0]
scenario = module
table = status.shared if step["scope"] == "shared" else status.extras
continue
if scenario is None:
continue
if result := RESULT.match(line):
counts = status._counts(table, scenario, "kde")
kind = result["kind"].strip()
if kind == "ok":
counts.passed += 1
elif kind == "FAIL":
counts.failed += 1
elif kind == "skip":
counts.skipped += 1
else:
counts.known += 1
def read_windows(status: Status, directory: Path) -> None:
"""Adds the latest `result.json` of each scenario under `directory`, by modification time."""
latest: dict[str, tuple[float, dict[str, Any]]] = {}
for path in directory.rglob("result.json"):
try:
data = json.loads(path.read_text())
except (OSError, ValueError):
continue
name = data.get("scenario")
if isinstance(name, str) and path.stat().st_mtime >= latest.get(name, (0.0, {}))[0]:
latest[name] = (path.stat().st_mtime, data)
for name, (_, data) in latest.items():
checks = [c for c in data.get("checks", []) if isinstance(c, dict)]
if not any(c.get("scope") for c in checks):
# a result with no scopes: the totals are all there is
counts = status._counts(status.shared, name, "windows")
counts.passed = int(data.get("passed", 0))
counts.failed = int(data.get("failed", 0))
counts.skipped = int(data.get("skipped", 0))
counts.known = int(data.get("known_failures", 0))
continue
for check in checks:
scope = check.get("scope")
if not scope:
continue # what a scenario needed to start, not what it checks
if scope == "shared" and not name.startswith("windows_"):
counts = status._counts(status.shared, name, "windows")
elif scope == "shared":
continue # the shared steps an extra's run starts from belong to their scenarios
else:
counts = status._counts(status.extras, name, "windows")
if check.get("skipped"):
counts.skipped += 1
elif check.get("known_failure"):
counts.known += 1
elif check.get("passed"):
counts.passed += 1
else:
counts.failed += 1
def _cell(counts: Counts) -> str:
return counts.cell() if counts.passed + counts.failed + counts.known + counts.skipped else "-"
def table(status: Status) -> str:
"""The Markdown tables: the shared scenarios on each port, then the extras of each."""
ports = ("kde", "windows")
out = ["| scenario | kde | windows |", "|---|---|---|"]
for name in sorted(status.shared):
cells = [status.shared[name].get(port, Counts()) for port in ports]
out.append(
f"| `{name}` | "
+ " | ".join(
c.cell() if c.passed + c.failed + c.known + c.skipped else "not run" for c in cells
)
+ " |"
)
out += ["", "Only on one port:", "", "| extra | kde | windows |", "|---|---|---|"]
for name in sorted(status.extras):
cells = [status.extras[name].get(port, Counts()) for port in ports]
out.append(f"| `{name}` | " + " | ".join(_cell(c) for c in cells) + " |")
return "\n".join(out) + "\n"
BEGIN = "<!-- scenario-status:begin -->"
END = "<!-- scenario-status:end -->"
def replace_region(document: str, content: str) -> str:
"""The document with what is between the two markers replaced by `content`."""
start, end = document.index(BEGIN) + len(BEGIN), document.index(END)
return f"{document[:start]}\n{content}{document[end:]}"
+67
View File
@@ -0,0 +1,67 @@
from pathlib import Path
from komorebi_vm import status
KDE_LOG = """\
vm-test-run-komorebi-x> == [kde] core_e2e.start
vm-test-run-komorebi-x> ok [kde] the daemon loaded its script
vm-test-run-komorebi-x> == [shared] tiling.run
vm-test-run-komorebi-x> ok [shared] every window is where the layout says
vm-test-run-komorebi-x> FAIL [shared] the windows share the work area
vm-test-run-komorebi-x> [shared] tiling.run: 3.2 s
vm-test-run-komorebi-x> == [shared] focus.run: left out by KFDE_ONLY
vm-test-run-komorebi-x> ok [shared] not counted: the step was left out
vm-test-run-komorebi-x> == [shared] commands.unknown_monitor
vm-test-run-komorebi-x> known failure [shared] a window is not lost: the container is removed
vm-test-run-komorebi-x> skip [shared] change-layout x: the port has no such layout
"""
def test_kde_counts_are_by_scenario_and_scope() -> None:
found = status.Status()
status.read_kde(found, KDE_LOG)
tiling = found.shared["tiling"]["kde"]
assert (tiling.passed, tiling.failed) == (1, 1)
assert "focus" not in found.shared
unknown = found.shared["commands"]["kde"]
assert (unknown.known, unknown.skipped) == (1, 1)
assert found.extras["core_e2e"]["kde"].passed == 1
def test_windows_counts_only_the_shared_checks_of_a_scenario(tmp_path: Path) -> None:
(tmp_path / "a").mkdir()
(tmp_path / "a" / "result.json").write_text(
'{"scenario": "closing", "checks": ['
'{"check": "setup", "passed": true},'
'{"check": "one", "passed": true, "scope": "shared"},'
'{"check": "two", "passed": false, "scope": "shared"},'
'{"check": "three", "passed": true, "scope": "windows"}]}'
)
found = status.Status()
status.read_windows(found, tmp_path)
closing = found.shared["closing"]["windows"]
assert (closing.passed, closing.failed) == (1, 1)
assert found.extras["closing"]["windows"].passed == 1
def test_a_windows_result_without_scopes_is_its_totals(tmp_path: Path) -> None:
(tmp_path / "result.json").write_text(
'{"scenario": "tiling", "passed": 10, "failed": 0, "skipped": 0, "known_failures": 0}'
)
found = status.Status()
status.read_windows(found, tmp_path)
assert found.shared["tiling"]["windows"].passed == 10
def test_the_table_has_a_row_for_each_scenario_and_a_dash_for_a_port_that_did_not_run() -> None:
found = status.Status()
status.read_kde(found, KDE_LOG)
out = status.table(found)
assert "| `tiling` | FAIL 1 of 2 | not run |" in out or "| `tiling` | FAIL 1 of 2 | - |" in out
def test_the_region_between_the_markers_is_replaced() -> None:
document = f"before\n{status.BEGIN}\nold\n{status.END}\nafter\n"
assert status.replace_region(document, "new\n") == (
f"before\n{status.BEGIN}\nnew\n{status.END}\nafter\n"
)
+36
View File
@@ -0,0 +1,36 @@
#!/usr/bin/env python3
"""Makes the scenario-by-port status table from the logs of runs on KDE and the result files of
kfw (see nix/tests/vm/lib/komorebi_vm/status.py).
scripts/scenario-status --kde LOG... [--windows DIR...] [--update docs/dev/BEHAVIOUR.md]
Without --update the table is printed. With it, what is between the two comment markers
(`<!-- scenario-status:begin -->` and `<!-- scenario-status:end -->`) of the file is replaced.
"""
import argparse
import importlib.util
from pathlib import Path
path = Path(__file__).resolve().parent.parent / "nix/tests/vm/lib/komorebi_vm/status.py"
spec = importlib.util.spec_from_file_location("status", path)
assert spec and spec.loader
status = importlib.util.module_from_spec(spec)
spec.loader.exec_module(status)
parser = argparse.ArgumentParser(description=__doc__.split("\n")[0])
parser.add_argument("--kde", nargs="*", default=[], type=Path, help="logs of runs on KDE")
parser.add_argument("--windows", nargs="*", default=[], type=Path, help="directories of kfw results")
parser.add_argument("--update", type=Path, help="the document to put the table in")
args = parser.parse_args()
found = status.Status()
for log in args.kde:
status.read_kde(found, log.read_text(errors="replace"))
for directory in args.windows:
status.read_windows(found, directory)
text = status.table(found)
if args.update:
args.update.write_text(status.replace_region(args.update.read_text(), text))
else:
print(text, end="")