mirror of
https://github.com/LGUG2Z/komorebi.git
synced 2026-11-16 13:51:19 +01:00
refactor(e2e): lift what a runner needs of a port out of the Windows runner
RecordedChecks (the checks a runner keeps for its result file), the step loop with its timings and known failures, the commit under test and the step keyword helper have nothing to do with Windows. Move them to komorebi_vm/runner.py, with the known-failure tables passed in, so that a runner of another port uses them instead of importing a sibling port or copying them. The Windows runner keeps its tables and the names it and its tests use. The unit tests of the core and of the Windows runner pass unchanged (163), and ruff reports what it did before.
This commit is contained in:
1 parent
f4715a3465
commit
ceaea8ca1b
2 files changed
+155
-116
No files matched your search
@@ -0,0 +1,145 @@
|
||||
"""What a runner of a port needs besides the scenarios: the checks it keeps for the result file,
|
||||
the steps it runs one at a time with their timings, and the commit under test.
|
||||
|
||||
Nothing here knows a port. A port says which steps are known failures (and which commit fixed
|
||||
what) to `run_steps`, and keeps its own tables.
|
||||
"""
|
||||
|
||||
import os
|
||||
import subprocess
|
||||
import time
|
||||
from collections.abc import Callable
|
||||
from pathlib import Path
|
||||
|
||||
from komorebi_vm.core.checks import Checks
|
||||
from komorebi_vm.core.suite import run_suite
|
||||
|
||||
|
||||
class RecordedChecks(Checks):
|
||||
"""`Checks` that also keeps each check, for the result file."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
self.results: list[dict[str, object]] = []
|
||||
# how long each step of the suite took, for the result file
|
||||
self.steps: list[dict[str, object]] = []
|
||||
# the reason the failing checks of the step being run are known ones
|
||||
self.expected: str | None = None
|
||||
|
||||
def _record(self, **fields: object) -> None:
|
||||
if self.scope:
|
||||
fields["scope"] = self.scope
|
||||
self.results.append(fields)
|
||||
|
||||
def ok(self, description: str) -> None:
|
||||
super().ok(description)
|
||||
self._record(check=description, passed=True)
|
||||
|
||||
def bad(self, description: str) -> None:
|
||||
if self.expected:
|
||||
self.known.append(f"{self.labelled(description)} ({self.expected})")
|
||||
print(f" known failure {self.labelled(description)}: {self.expected}")
|
||||
self._record(
|
||||
check=description, known_failure=True, reason=self.expected
|
||||
)
|
||||
return
|
||||
super().bad(description)
|
||||
self._record(check=description, passed=False)
|
||||
|
||||
def skip(self, description: str, reason: str) -> None:
|
||||
super().skip(description, reason)
|
||||
self._record(check=description, skipped=True, reason=reason)
|
||||
|
||||
def known_failure(
|
||||
self, description: str, condition: bool, reason: str
|
||||
) -> bool:
|
||||
holds = super().known_failure(description, condition, reason)
|
||||
if not holds:
|
||||
self._record(
|
||||
check=description, known_failure=True, reason=reason
|
||||
)
|
||||
return holds
|
||||
|
||||
|
||||
|
||||
def run_steps(
|
||||
checks: "RecordedChecks",
|
||||
*steps,
|
||||
known: dict[str, str] | None = None,
|
||||
fixed_by: dict[str, tuple[str, str | None]] | None = None,
|
||||
contains: Callable[[str], bool] | None = None,
|
||||
):
|
||||
"""`run_suite`, one step at a time, keeping how long each took.
|
||||
|
||||
`known` maps a step to the reason its failing checks are known failures, and `fixed_by` maps a
|
||||
step to the commit that fixed it and the reason that still holds after it (`None`: nothing);
|
||||
`contains` says whether the commit under test has a fix."""
|
||||
known = known or {}
|
||||
fixed_by = fixed_by or {}
|
||||
results: list[object] = []
|
||||
for step in steps:
|
||||
began = time.monotonic()
|
||||
counted = checks.passed + checks.failed + len(checks.known)
|
||||
fixed = fixed_by.get(step.name)
|
||||
checks.expected = (
|
||||
fixed[1] if fixed and contains and contains(fixed[0])
|
||||
else known.get(step.name)
|
||||
)
|
||||
known_before = len(checks.known)
|
||||
try:
|
||||
results.extend(run_suite(checks, step))
|
||||
finally:
|
||||
reason, checks.expected = checks.expected, None
|
||||
ran = checks.passed + checks.failed + len(checks.known) > counted
|
||||
if reason and ran and len(checks.known) == known_before:
|
||||
checks.bad(
|
||||
f"{step.name}: nothing failed, so it is no longer a known "
|
||||
"failure"
|
||||
)
|
||||
checks.steps.append({
|
||||
"step": f"[{step.scope}] {step.name}",
|
||||
"seconds": round(time.monotonic() - began, 1),
|
||||
})
|
||||
return results
|
||||
|
||||
|
||||
def with_kwargs(fn, **kwargs):
|
||||
"""`fn` with keyword arguments, for a suite step, which takes positional
|
||||
ones only; it keeps the name the step prints."""
|
||||
def step(*args):
|
||||
return fn(*args, **kwargs)
|
||||
|
||||
step.__module__ = fn.__module__
|
||||
step.__name__ = fn.__name__
|
||||
return step
|
||||
|
||||
|
||||
def contains(fix: str, repo: Path | None = None) -> bool:
|
||||
"""Whether the commit under test has the commit `fix` in its history, so that
|
||||
a marker for the bug it fixed no longer applies. Not known: no."""
|
||||
commit = os.environ.get("KOMOREBI_E2E_COMMIT")
|
||||
if not commit:
|
||||
return False
|
||||
here = repo or Path(__file__).resolve().parent
|
||||
git = subprocess.run(
|
||||
["git", "-C", str(here), "merge-base", "--is-ancestor", fix, commit],
|
||||
capture_output=True,
|
||||
)
|
||||
return git.returncode == 0
|
||||
|
||||
|
||||
|
||||
def commit() -> str:
|
||||
"""The kfw commit under test: set `KOMOREBI_E2E_COMMIT` to the commit the
|
||||
binaries were built from; otherwise this tree's HEAD, which may not be it."""
|
||||
if os.environ.get("KOMOREBI_E2E_COMMIT"):
|
||||
return os.environ["KOMOREBI_E2E_COMMIT"]
|
||||
here = Path(__file__).resolve().parent
|
||||
try:
|
||||
git = subprocess.run(
|
||||
["git", "-C", str(here), "rev-parse", "HEAD"],
|
||||
capture_output=True, text=True,
|
||||
)
|
||||
except FileNotFoundError:
|
||||
return "unknown"
|
||||
return git.stdout.strip() or "unknown"
|
||||
@@ -29,15 +29,17 @@ another (state-kfw-stack.json) and after the focused one is floated
|
||||
import importlib
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import traceback
|
||||
import zipfile
|
||||
from pathlib import Path
|
||||
|
||||
from komorebi_vm.core.checks import Checks
|
||||
from komorebi_vm.core.suite import platform_only, run_suite, shared
|
||||
from komorebi_vm.core.suite import platform_only, shared
|
||||
from komorebi_vm.runner import RecordedChecks, run_steps
|
||||
from komorebi_vm.runner import commit as _commit
|
||||
from komorebi_vm.runner import contains as _contains
|
||||
from komorebi_vm.runner import with_kwargs as _with
|
||||
|
||||
from .agent import AgentError, AgentLost
|
||||
from .guest import Guest
|
||||
@@ -91,104 +93,12 @@ class GuestDied(RuntimeError):
|
||||
"""QEMU exited while a scenario was running: not what kfw did."""
|
||||
|
||||
|
||||
class RecordedChecks(Checks):
|
||||
"""`Checks` that also keeps each check, for the result file."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
self.results: list[dict[str, object]] = []
|
||||
# how long each step of the suite took, for the result file
|
||||
self.steps: list[dict[str, object]] = []
|
||||
# the reason the failing checks of the step being run are known ones
|
||||
self.expected: str | None = None
|
||||
|
||||
def _record(self, **fields: object) -> None:
|
||||
if self.scope:
|
||||
fields["scope"] = self.scope
|
||||
self.results.append(fields)
|
||||
|
||||
def ok(self, description: str) -> None:
|
||||
super().ok(description)
|
||||
self._record(check=description, passed=True)
|
||||
|
||||
def bad(self, description: str) -> None:
|
||||
if self.expected:
|
||||
self.known.append(f"{self.labelled(description)} ({self.expected})")
|
||||
print(f" known failure {self.labelled(description)}: {self.expected}")
|
||||
self._record(
|
||||
check=description, known_failure=True, reason=self.expected
|
||||
)
|
||||
return
|
||||
super().bad(description)
|
||||
self._record(check=description, passed=False)
|
||||
|
||||
def skip(self, description: str, reason: str) -> None:
|
||||
super().skip(description, reason)
|
||||
self._record(check=description, skipped=True, reason=reason)
|
||||
|
||||
def known_failure(
|
||||
self, description: str, condition: bool, reason: str
|
||||
) -> bool:
|
||||
holds = super().known_failure(description, condition, reason)
|
||||
if not holds:
|
||||
self._record(
|
||||
check=description, known_failure=True, reason=reason
|
||||
)
|
||||
return holds
|
||||
|
||||
|
||||
def _suite(checks: "RecordedChecks", *steps):
|
||||
"""`run_suite`, one step at a time, keeping how long each took."""
|
||||
results: list[object] = []
|
||||
for step in steps:
|
||||
began = time.monotonic()
|
||||
counted = checks.passed + checks.failed + len(checks.known)
|
||||
fixed = KNOWN_STEPS_FIXED_BY.get(step.name)
|
||||
checks.expected = (
|
||||
fixed[1] if fixed and _contains(fixed[0])
|
||||
else KNOWN_STEPS.get(step.name)
|
||||
)
|
||||
known_before = len(checks.known)
|
||||
try:
|
||||
results.extend(run_suite(checks, step))
|
||||
finally:
|
||||
reason, checks.expected = checks.expected, None
|
||||
ran = checks.passed + checks.failed + len(checks.known) > counted
|
||||
if reason and ran and len(checks.known) == known_before:
|
||||
checks.bad(
|
||||
f"{step.name}: nothing failed, so it is no longer a known "
|
||||
"failure"
|
||||
)
|
||||
checks.steps.append({
|
||||
"step": f"[{step.scope}] {step.name}",
|
||||
"seconds": round(time.monotonic() - began, 1),
|
||||
})
|
||||
return results
|
||||
|
||||
|
||||
def _with(fn, **kwargs):
|
||||
"""`fn` with keyword arguments, for a suite step, which takes positional
|
||||
ones only; it keeps the name the step prints."""
|
||||
def step(*args):
|
||||
return fn(*args, **kwargs)
|
||||
|
||||
step.__module__ = fn.__module__
|
||||
step.__name__ = fn.__name__
|
||||
return step
|
||||
|
||||
|
||||
def _contains(fix: str, repo: Path | None = None) -> bool:
|
||||
"""Whether the commit under test has the commit `fix` in its history, so that
|
||||
a marker for the bug it fixed no longer applies. Not known: no."""
|
||||
commit = os.environ.get("KOMOREBI_E2E_COMMIT")
|
||||
if not commit:
|
||||
return False
|
||||
here = repo or Path(__file__).resolve().parent
|
||||
git = subprocess.run(
|
||||
["git", "-C", str(here), "merge-base", "--is-ancestor", fix, commit],
|
||||
capture_output=True,
|
||||
def _suite(checks: RecordedChecks, *steps):
|
||||
"""`run_steps` with the known failures of this port."""
|
||||
return run_steps(
|
||||
checks, *steps, known=KNOWN_STEPS, fixed_by=KNOWN_STEPS_FIXED_BY,
|
||||
contains=_contains,
|
||||
)
|
||||
return git.returncode == 0
|
||||
|
||||
|
||||
# the commit that fixed each bug a marker is for (komorebi's `next`)
|
||||
@@ -196,22 +106,6 @@ UNKNOWN_MONITOR_FIX = "837ca257"
|
||||
PLACEMENT_INTEGER_FIX = "3b424dd7"
|
||||
|
||||
|
||||
def _commit() -> str:
|
||||
"""The kfw commit under test: set `KOMOREBI_E2E_COMMIT` to the commit the
|
||||
binaries were built from; otherwise this tree's HEAD, which may not be it."""
|
||||
if os.environ.get("KOMOREBI_E2E_COMMIT"):
|
||||
return os.environ["KOMOREBI_E2E_COMMIT"]
|
||||
here = Path(__file__).resolve().parent
|
||||
try:
|
||||
git = subprocess.run(
|
||||
["git", "-C", str(here), "rev-parse", "HEAD"],
|
||||
capture_output=True, text=True,
|
||||
)
|
||||
except FileNotFoundError:
|
||||
return "unknown"
|
||||
return git.stdout.strip() or "unknown"
|
||||
|
||||
|
||||
BINARIES = ["komorebi.exe", "komorebic.exe", "komorebi-testwin.exe"]
|
||||
# uploaded when the binaries have them: only the `bar` scenario needs the bar
|
||||
OPTIONAL_BINARIES = ["komorebi-bar.exe"]
|
||||
|
||||
Reference in new issue
Block a user