"""What a runner of a port needs besides the scenarios: the checks it keeps for the result file, the steps it runs one at a time with their timings, and the commit under test. Nothing here knows a port. A port says which steps are known failures (and which commit fixed what) to `run_steps`, and keeps its own tables. """ import os import subprocess import time from collections.abc import Callable from pathlib import Path from komorebi_vm.core.checks import Checks from komorebi_vm.core.suite import run_suite class RecordedChecks(Checks): """`Checks` that also keeps each check, for the result file.""" def __init__(self, known_checks: dict[str, str] | None = None) -> None: super().__init__() self.results: list[dict[str, object]] = [] # how long each step of the suite took, for the result file self.steps: list[dict[str, object]] = [] # the reason the failing checks of the step being run are known ones self.expected: str | None = None # a part of the description of a check, and the reason it is a known failure of the port: a # finer mark than a step, for a scenario of which only some checks are about the difference self.known_checks = known_checks or {} # the marks whose check passed, which a run says can go self.known_passed: list[str] = [] def _known_check(self, description: str) -> tuple[str, str] | None: for key, reason in self.known_checks.items(): if key in description: return key, reason return None def stale_marks(self) -> None: """Fails once for each known-check mark whose check passed: the difference is gone.""" for key in self.known_passed: description = f"known-failure mark no longer needed, its check passes: {key}" # not `self.bad`: this text holds the mark's own words, and would be taken for the # check it is about Checks.bad(self, description) self._record(check=description, passed=False) def _record(self, **fields: object) -> None: if self.scope: fields["scope"] = self.scope self.results.append(fields) def ok(self, description: str) -> None: super().ok(description) self._record(check=description, passed=True) if (mark := self._known_check(description)) and mark[0] not in self.known_passed: self.known_passed.append(mark[0]) def bad(self, description: str) -> None: mark = self._known_check(description) reason = self.expected or (mark[1] if mark else None) if reason: self.known.append(f"{self.labelled(description)} ({reason})") print(f" known failure {self.labelled(description)}: {reason}") self._record(check=description, known_failure=True, reason=reason) return super().bad(description) self._record(check=description, passed=False) def skip(self, description: str, reason: str) -> None: super().skip(description, reason) self._record(check=description, skipped=True, reason=reason) def known_failure( self, description: str, condition: bool, reason: str ) -> bool: holds = super().known_failure(description, condition, reason) if not holds: self._record( check=description, known_failure=True, reason=reason ) return holds def run_steps( checks: "RecordedChecks", *steps, known: dict[str, str] | None = None, fixed_by: dict[str, tuple[str, str | None]] | None = None, contains: Callable[[str], bool] | None = None, ): """`run_suite`, one step at a time, keeping how long each took. `known` maps a step to the reason its failing checks are known failures, and `fixed_by` maps a step to the commit that fixed it and the reason that still holds after it (`None`: nothing); `contains` says whether the commit under test has a fix.""" known = known or {} fixed_by = fixed_by or {} results: list[object] = [] for step in steps: began = time.monotonic() counted = checks.passed + checks.failed + len(checks.known) fixed = fixed_by.get(step.name) checks.expected = ( fixed[1] if fixed and contains and contains(fixed[0]) else known.get(step.name) ) known_before = len(checks.known) try: results.extend(run_suite(checks, step)) finally: reason, checks.expected = checks.expected, None ran = checks.passed + checks.failed + len(checks.known) > counted if reason and ran and len(checks.known) == known_before: checks.bad( f"{step.name}: nothing failed, so it is no longer a known " "failure" ) checks.steps.append({ "step": f"[{step.scope}] {step.name}", "seconds": round(time.monotonic() - began, 1), }) return results def with_kwargs(fn, **kwargs): """`fn` with keyword arguments, for a suite step, which takes positional ones only; it keeps the name the step prints.""" def step(*args): return fn(*args, **kwargs) step.__module__ = fn.__module__ step.__name__ = fn.__name__ return step def contains(fix: str, repo: Path | None = None) -> bool: """Whether the commit under test has the commit `fix` in its history, so that a marker for the bug it fixed no longer applies. Not known: no.""" commit = os.environ.get("KOMOREBI_E2E_COMMIT") if not commit: return False here = repo or Path(__file__).resolve().parent git = subprocess.run( ["git", "-C", str(here), "merge-base", "--is-ancestor", fix, commit], capture_output=True, check=False, ) return git.returncode == 0 def commit() -> str: """The kfw commit under test: set `KOMOREBI_E2E_COMMIT` to the commit the binaries were built from; otherwise this tree's HEAD, which may not be it.""" if os.environ.get("KOMOREBI_E2E_COMMIT"): return os.environ["KOMOREBI_E2E_COMMIT"] here = Path(__file__).resolve().parent try: git = subprocess.run( ["git", "-C", str(here), "rev-parse", "HEAD"], capture_output=True, text=True, check=False, ) except FileNotFoundError: return "unknown" return git.stdout.strip() or "unknown"