From 25f50076afe74e61ba1c1889653f2059cc5af77e Mon Sep 17 00:00:00 2001 From: LGUG2Z Date: Fri, 6 Nov 2026 13:37:11 -0800 Subject: [PATCH] feat(e2e): mark one check of a scenario as a known failure of a port A step could be marked as a known failure of a port, which hides every failing check of it: too coarse for a scenario in which one or two checks are about a difference of the port and the others are not. RecordedChecks takes marks now: a part of the description of a check, and the reason. A marked check that fails is recorded as a known failure, as a step is, and one that passes is a failure of its own, so that a mark does not outlive the difference it was for. A port that gives no marks is as it was. The run of the runner's own subprocess calls says it does not check them. --- e2e/lib/komorebi_vm/runner.py | 40 +++++++++++++---- e2e/lib/tests/test_recorded_checks.py | 62 +++++++++++++++++++++++++++ 2 files changed, 93 insertions(+), 9 deletions(-) create mode 100644 e2e/lib/tests/test_recorded_checks.py diff --git a/e2e/lib/komorebi_vm/runner.py b/e2e/lib/komorebi_vm/runner.py index 7be83eeb..420bfbf6 100644 --- a/e2e/lib/komorebi_vm/runner.py +++ b/e2e/lib/komorebi_vm/runner.py @@ -18,13 +18,33 @@ from komorebi_vm.core.suite import run_suite class RecordedChecks(Checks): """`Checks` that also keeps each check, for the result file.""" - def __init__(self) -> None: + def __init__(self, known_checks: dict[str, str] | None = None) -> None: super().__init__() self.results: list[dict[str, object]] = [] # how long each step of the suite took, for the result file self.steps: list[dict[str, object]] = [] # the reason the failing checks of the step being run are known ones self.expected: str | None = None + # a part of the description of a check, and the reason it is a known failure of the port: a + # finer mark than a step, for a scenario of which only some checks are about the difference + self.known_checks = known_checks or {} + # the marks whose check passed, which a run says can go + self.known_passed: list[str] = [] + + def _known_check(self, description: str) -> tuple[str, str] | None: + for key, reason in self.known_checks.items(): + if key in description: + return key, reason + return None + + def stale_marks(self) -> None: + """Fails once for each known-check mark whose check passed: the difference is gone.""" + for key in self.known_passed: + description = f"known-failure mark no longer needed, its check passes: {key}" + # not `self.bad`: this text holds the mark's own words, and would be taken for the + # check it is about + Checks.bad(self, description) + self._record(check=description, passed=False) def _record(self, **fields: object) -> None: if self.scope: @@ -34,14 +54,16 @@ class RecordedChecks(Checks): def ok(self, description: str) -> None: super().ok(description) self._record(check=description, passed=True) + if (mark := self._known_check(description)) and mark[0] not in self.known_passed: + self.known_passed.append(mark[0]) def bad(self, description: str) -> None: - if self.expected: - self.known.append(f"{self.labelled(description)} ({self.expected})") - print(f" known failure {self.labelled(description)}: {self.expected}") - self._record( - check=description, known_failure=True, reason=self.expected - ) + mark = self._known_check(description) + reason = self.expected or (mark[1] if mark else None) + if reason: + self.known.append(f"{self.labelled(description)} ({reason})") + print(f" known failure {self.labelled(description)}: {reason}") + self._record(check=description, known_failure=True, reason=reason) return super().bad(description) self._record(check=description, passed=False) @@ -123,7 +145,7 @@ def contains(fix: str, repo: Path | None = None) -> bool: here = repo or Path(__file__).resolve().parent git = subprocess.run( ["git", "-C", str(here), "merge-base", "--is-ancestor", fix, commit], - capture_output=True, + capture_output=True, check=False, ) return git.returncode == 0 @@ -138,7 +160,7 @@ def commit() -> str: try: git = subprocess.run( ["git", "-C", str(here), "rev-parse", "HEAD"], - capture_output=True, text=True, + capture_output=True, text=True, check=False, ) except FileNotFoundError: return "unknown" diff --git a/e2e/lib/tests/test_recorded_checks.py b/e2e/lib/tests/test_recorded_checks.py new file mode 100644 index 00000000..e4f4c614 --- /dev/null +++ b/e2e/lib/tests/test_recorded_checks.py @@ -0,0 +1,62 @@ +"""The checks a runner keeps, and the marks of the checks that are known failures of a port.""" + +from komorebi_vm.runner import RecordedChecks + +MARKS = {"covers the display": "the daemon has no fullscreen"} + + +def test_a_marked_check_that_fails_is_a_known_failure_and_not_a_failure() -> None: + checks = RecordedChecks(MARKS) + checks.bad("the window covers the display") + assert checks.failed == 0 + assert checks.known == ["the window covers the display (the daemon has no fullscreen)"] + assert checks.results == [ + { + "check": "the window covers the display", + "known_failure": True, + "reason": "the daemon has no fullscreen", + } + ] + + +def test_a_check_that_is_not_marked_fails() -> None: + checks = RecordedChecks(MARKS) + checks.bad("another check") + assert checks.failed == 1 and checks.known == [] + + +def test_no_marks_is_how_it_was() -> None: + checks = RecordedChecks() + checks.bad("the window covers the display") + assert checks.failed == 1 and checks.known == [] + + +def test_a_marked_check_that_passes_says_its_mark_can_go() -> None: + checks = RecordedChecks(MARKS) + checks.ok("the window covers the display") + assert checks.failed == 0 + checks.stale_marks() + assert checks.failed == 1 + assert "covers the display" in checks.failures[0] + + +def test_a_mark_whose_check_never_ran_is_not_stale() -> None: + checks = RecordedChecks(MARKS) + checks.ok("something else") + checks.stale_marks() + assert checks.failed == 0 + + +def test_a_mark_is_said_stale_once() -> None: + checks = RecordedChecks(MARKS) + checks.ok("the window covers the display") + checks.ok("the second window covers the display") + checks.stale_marks() + assert checks.failed == 1 + + +def test_the_reason_of_a_step_is_kept_over_a_mark() -> None: + checks = RecordedChecks(MARKS) + checks.expected = "the step is known" + checks.bad("the window covers the display") + assert checks.known == ["the window covers the display (the step is known)"]