mirror of
https://github.com/LGUG2Z/komorebi.git
synced 2026-11-16 05:41:16 +01:00
feat(repo): scenario-status makes the table from the runs
This commit is contained in:
1 parent
781b082778
commit
37bb9e3fe4
3 files changed
+270
No files matched your search
@@ -0,0 +1,167 @@
|
||||
"""The scenario-by-port status table, from the runs themselves.
|
||||
|
||||
KDE prints `== [scope] module.function` before the checks of each step of a suite, and each check
|
||||
as ` ok [scope] ...`, ` FAIL ...`, ` skip ...` or ` known failure ...`; the log of a
|
||||
`nix build -L` has a prefix on every line, which is dropped. kfw writes a `result.json` for each
|
||||
scenario, with the passed, failed, skipped and known checks, each with its scope. This reads both
|
||||
and makes one row for each scenario (the module name: `rules_exe`, `commands`) and one column for
|
||||
each port, so that the table in docs/dev/BEHAVIOUR.md says what the last run said and not what
|
||||
somebody wrote. Only the checks labelled `shared` are the scenario's; what a platform labelled
|
||||
with its own name is an extra, and what a scenario needed to start has no label and is left out.
|
||||
|
||||
Only the standard library: `scripts/scenario-status` loads this file by its path.
|
||||
"""
|
||||
|
||||
import json
|
||||
import re
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
PREFIX = re.compile(r"^[\w.-]+> ")
|
||||
STEP = re.compile(r"^== \[(?P<scope>[\w-]+)\] (?P<name>[\w.]+)(?P<left_out>: left out by \w+)?\s*$")
|
||||
RESULT = re.compile(r"^ (?P<kind>ok |FAIL|skip|known failure) ")
|
||||
|
||||
|
||||
@dataclass
|
||||
class Counts:
|
||||
passed: int = 0
|
||||
failed: int = 0
|
||||
skipped: int = 0
|
||||
known: int = 0
|
||||
|
||||
def add(self, other: "Counts") -> None:
|
||||
self.passed += other.passed
|
||||
self.failed += other.failed
|
||||
self.skipped += other.skipped
|
||||
self.known += other.known
|
||||
|
||||
def cell(self) -> str:
|
||||
"""`pass 9`, `FAIL 2 of 9`, with `, 1 known` and `, 1 skipped` after."""
|
||||
total = self.passed + self.failed
|
||||
head = f"pass {self.passed}" if not self.failed else f"FAIL {self.failed} of {total}"
|
||||
if not total and not (self.known or self.skipped):
|
||||
head = "-"
|
||||
parts = [head]
|
||||
if self.known:
|
||||
parts.append(f"{self.known} known")
|
||||
if self.skipped:
|
||||
parts.append(f"{self.skipped} skipped")
|
||||
return ", ".join(parts)
|
||||
|
||||
|
||||
@dataclass
|
||||
class Status:
|
||||
"""The counts of each scenario, for each port."""
|
||||
|
||||
shared: dict[str, dict[str, Counts]] = field(default_factory=dict)
|
||||
"""scenario -> port -> counts, for the steps labelled `shared` (and every kfw scenario)."""
|
||||
extras: dict[str, dict[str, Counts]] = field(default_factory=dict)
|
||||
"""extra -> port -> counts, for the steps a platform labelled with its own name."""
|
||||
|
||||
def _counts(self, table: dict[str, dict[str, Counts]], name: str, port: str) -> Counts:
|
||||
return table.setdefault(name, {}).setdefault(port, Counts())
|
||||
|
||||
|
||||
def read_kde(status: Status, text: str) -> None:
|
||||
"""Adds the checks of a log of a run on KDE."""
|
||||
scenario: str | None = None
|
||||
table = status.shared
|
||||
for raw in text.splitlines():
|
||||
line = PREFIX.sub("", raw)
|
||||
if step := STEP.match(line):
|
||||
if step["left_out"]:
|
||||
scenario = None
|
||||
continue
|
||||
module = step["name"].split(".")[0]
|
||||
scenario = module
|
||||
table = status.shared if step["scope"] == "shared" else status.extras
|
||||
continue
|
||||
if scenario is None:
|
||||
continue
|
||||
if result := RESULT.match(line):
|
||||
counts = status._counts(table, scenario, "kde")
|
||||
kind = result["kind"].strip()
|
||||
if kind == "ok":
|
||||
counts.passed += 1
|
||||
elif kind == "FAIL":
|
||||
counts.failed += 1
|
||||
elif kind == "skip":
|
||||
counts.skipped += 1
|
||||
else:
|
||||
counts.known += 1
|
||||
|
||||
|
||||
def read_windows(status: Status, directory: Path) -> None:
|
||||
"""Adds the latest `result.json` of each scenario under `directory`, by modification time."""
|
||||
latest: dict[str, tuple[float, dict[str, Any]]] = {}
|
||||
for path in directory.rglob("result.json"):
|
||||
try:
|
||||
data = json.loads(path.read_text())
|
||||
except (OSError, ValueError):
|
||||
continue
|
||||
name = data.get("scenario")
|
||||
if isinstance(name, str) and path.stat().st_mtime >= latest.get(name, (0.0, {}))[0]:
|
||||
latest[name] = (path.stat().st_mtime, data)
|
||||
for name, (_, data) in latest.items():
|
||||
checks = [c for c in data.get("checks", []) if isinstance(c, dict)]
|
||||
if not any(c.get("scope") for c in checks):
|
||||
# a result with no scopes: the totals are all there is
|
||||
counts = status._counts(status.shared, name, "windows")
|
||||
counts.passed = int(data.get("passed", 0))
|
||||
counts.failed = int(data.get("failed", 0))
|
||||
counts.skipped = int(data.get("skipped", 0))
|
||||
counts.known = int(data.get("known_failures", 0))
|
||||
continue
|
||||
for check in checks:
|
||||
scope = check.get("scope")
|
||||
if not scope:
|
||||
continue # what a scenario needed to start, not what it checks
|
||||
if scope == "shared" and not name.startswith("windows_"):
|
||||
counts = status._counts(status.shared, name, "windows")
|
||||
elif scope == "shared":
|
||||
continue # the shared steps an extra's run starts from belong to their scenarios
|
||||
else:
|
||||
counts = status._counts(status.extras, name, "windows")
|
||||
if check.get("skipped"):
|
||||
counts.skipped += 1
|
||||
elif check.get("known_failure"):
|
||||
counts.known += 1
|
||||
elif check.get("passed"):
|
||||
counts.passed += 1
|
||||
else:
|
||||
counts.failed += 1
|
||||
|
||||
|
||||
def _cell(counts: Counts) -> str:
|
||||
return counts.cell() if counts.passed + counts.failed + counts.known + counts.skipped else "-"
|
||||
|
||||
|
||||
def table(status: Status) -> str:
|
||||
"""The Markdown tables: the shared scenarios on each port, then the extras of each."""
|
||||
ports = ("kde", "windows")
|
||||
out = ["| scenario | kde | windows |", "|---|---|---|"]
|
||||
for name in sorted(status.shared):
|
||||
cells = [status.shared[name].get(port, Counts()) for port in ports]
|
||||
out.append(
|
||||
f"| `{name}` | "
|
||||
+ " | ".join(
|
||||
c.cell() if c.passed + c.failed + c.known + c.skipped else "not run" for c in cells
|
||||
)
|
||||
+ " |"
|
||||
)
|
||||
out += ["", "Only on one port:", "", "| extra | kde | windows |", "|---|---|---|"]
|
||||
for name in sorted(status.extras):
|
||||
cells = [status.extras[name].get(port, Counts()) for port in ports]
|
||||
out.append(f"| `{name}` | " + " | ".join(_cell(c) for c in cells) + " |")
|
||||
return "\n".join(out) + "\n"
|
||||
|
||||
|
||||
BEGIN = "<!-- scenario-status:begin -->"
|
||||
END = "<!-- scenario-status:end -->"
|
||||
|
||||
|
||||
def replace_region(document: str, content: str) -> str:
|
||||
"""The document with what is between the two markers replaced by `content`."""
|
||||
start, end = document.index(BEGIN) + len(BEGIN), document.index(END)
|
||||
return f"{document[:start]}\n{content}{document[end:]}"
|
||||
@@ -0,0 +1,67 @@
|
||||
from pathlib import Path
|
||||
|
||||
from komorebi_vm import status
|
||||
|
||||
KDE_LOG = """\
|
||||
vm-test-run-komorebi-x> == [kde] core_e2e.start
|
||||
vm-test-run-komorebi-x> ok [kde] the daemon loaded its script
|
||||
vm-test-run-komorebi-x> == [shared] tiling.run
|
||||
vm-test-run-komorebi-x> ok [shared] every window is where the layout says
|
||||
vm-test-run-komorebi-x> FAIL [shared] the windows share the work area
|
||||
vm-test-run-komorebi-x> [shared] tiling.run: 3.2 s
|
||||
vm-test-run-komorebi-x> == [shared] focus.run: left out by KFDE_ONLY
|
||||
vm-test-run-komorebi-x> ok [shared] not counted: the step was left out
|
||||
vm-test-run-komorebi-x> == [shared] commands.unknown_monitor
|
||||
vm-test-run-komorebi-x> known failure [shared] a window is not lost: the container is removed
|
||||
vm-test-run-komorebi-x> skip [shared] change-layout x: the port has no such layout
|
||||
"""
|
||||
|
||||
|
||||
def test_kde_counts_are_by_scenario_and_scope() -> None:
|
||||
found = status.Status()
|
||||
status.read_kde(found, KDE_LOG)
|
||||
tiling = found.shared["tiling"]["kde"]
|
||||
assert (tiling.passed, tiling.failed) == (1, 1)
|
||||
assert "focus" not in found.shared
|
||||
unknown = found.shared["commands"]["kde"]
|
||||
assert (unknown.known, unknown.skipped) == (1, 1)
|
||||
assert found.extras["core_e2e"]["kde"].passed == 1
|
||||
|
||||
|
||||
def test_windows_counts_only_the_shared_checks_of_a_scenario(tmp_path: Path) -> None:
|
||||
(tmp_path / "a").mkdir()
|
||||
(tmp_path / "a" / "result.json").write_text(
|
||||
'{"scenario": "closing", "checks": ['
|
||||
'{"check": "setup", "passed": true},'
|
||||
'{"check": "one", "passed": true, "scope": "shared"},'
|
||||
'{"check": "two", "passed": false, "scope": "shared"},'
|
||||
'{"check": "three", "passed": true, "scope": "windows"}]}'
|
||||
)
|
||||
found = status.Status()
|
||||
status.read_windows(found, tmp_path)
|
||||
closing = found.shared["closing"]["windows"]
|
||||
assert (closing.passed, closing.failed) == (1, 1)
|
||||
assert found.extras["closing"]["windows"].passed == 1
|
||||
|
||||
|
||||
def test_a_windows_result_without_scopes_is_its_totals(tmp_path: Path) -> None:
|
||||
(tmp_path / "result.json").write_text(
|
||||
'{"scenario": "tiling", "passed": 10, "failed": 0, "skipped": 0, "known_failures": 0}'
|
||||
)
|
||||
found = status.Status()
|
||||
status.read_windows(found, tmp_path)
|
||||
assert found.shared["tiling"]["windows"].passed == 10
|
||||
|
||||
|
||||
def test_the_table_has_a_row_for_each_scenario_and_a_dash_for_a_port_that_did_not_run() -> None:
|
||||
found = status.Status()
|
||||
status.read_kde(found, KDE_LOG)
|
||||
out = status.table(found)
|
||||
assert "| `tiling` | FAIL 1 of 2 | not run |" in out or "| `tiling` | FAIL 1 of 2 | - |" in out
|
||||
|
||||
|
||||
def test_the_region_between_the_markers_is_replaced() -> None:
|
||||
document = f"before\n{status.BEGIN}\nold\n{status.END}\nafter\n"
|
||||
assert status.replace_region(document, "new\n") == (
|
||||
f"before\n{status.BEGIN}\nnew\n{status.END}\nafter\n"
|
||||
)
|
||||
Executable
+36
@@ -0,0 +1,36 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Makes the scenario-by-port status table from the logs of runs on KDE and the result files of
|
||||
kfw (see nix/tests/vm/lib/komorebi_vm/status.py).
|
||||
|
||||
scripts/scenario-status --kde LOG... [--windows DIR...] [--update docs/dev/BEHAVIOUR.md]
|
||||
|
||||
Without --update the table is printed. With it, what is between the two comment markers
|
||||
(`<!-- scenario-status:begin -->` and `<!-- scenario-status:end -->`) of the file is replaced.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import importlib.util
|
||||
from pathlib import Path
|
||||
|
||||
path = Path(__file__).resolve().parent.parent / "nix/tests/vm/lib/komorebi_vm/status.py"
|
||||
spec = importlib.util.spec_from_file_location("status", path)
|
||||
assert spec and spec.loader
|
||||
status = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(status)
|
||||
|
||||
parser = argparse.ArgumentParser(description=__doc__.split("\n")[0])
|
||||
parser.add_argument("--kde", nargs="*", default=[], type=Path, help="logs of runs on KDE")
|
||||
parser.add_argument("--windows", nargs="*", default=[], type=Path, help="directories of kfw results")
|
||||
parser.add_argument("--update", type=Path, help="the document to put the table in")
|
||||
args = parser.parse_args()
|
||||
|
||||
found = status.Status()
|
||||
for log in args.kde:
|
||||
status.read_kde(found, log.read_text(errors="replace"))
|
||||
for directory in args.windows:
|
||||
status.read_windows(found, directory)
|
||||
text = status.table(found)
|
||||
if args.update:
|
||||
args.update.write_text(status.replace_region(args.update.read_text(), text))
|
||||
else:
|
||||
print(text, end="")
|
||||
Reference in new issue
Block a user