diff --git a/e2e/lib/komorebi_vm/core/checks.py b/e2e/lib/komorebi_vm/core/checks.py index 627460a4..2f6dc923 100644 --- a/e2e/lib/komorebi_vm/core/checks.py +++ b/e2e/lib/komorebi_vm/core/checks.py @@ -16,6 +16,7 @@ class Checks: self.failed = 0 self.failures: list[str] = [] self.skipped: list[str] = [] + self.known: list[str] = [] def ok(self, description: str) -> None: """Records a check that passed.""" @@ -33,6 +34,19 @@ class Checks: self.skipped.append(f"{description} ({reason})") print(f" skip {description}: {reason}") + def known_failure(self, description: str, condition: bool, reason: str) -> bool: + """Records a check that is known to fail, with the reason, to follow up on later. + + The suite does not fail while it does. When it holds, that is a failure, to say that + the marker can go. + """ + if condition: + self.bad(f"{description}: it holds now, so it is no longer a known failure") + else: + self.known.append(f"{description} ({reason})") + print(f" known failure {description}: {reason}") + return condition + def check(self, description: str, condition: bool | Callable[[], bool]) -> bool: """Records `condition` (or what calling it returns) and returns it.""" holds = condition() if callable(condition) else condition @@ -51,5 +65,7 @@ class Checks: print(f"passed: {self.passed} failed: {self.failed}") if self.skipped: print(f"skipped: {len(self.skipped)}") + if self.known: + print(f"known failures: {len(self.known)}") if self.failed: raise AssertionError(f"{self.failed} checks failed: {'; '.join(self.failures)}") diff --git a/e2e/lib/komorebi_vm/core/scenarios/commands.py b/e2e/lib/komorebi_vm/core/scenarios/commands.py index 836e0859..23935f0a 100644 --- a/e2e/lib/komorebi_vm/core/scenarios/commands.py +++ b/e2e/lib/komorebi_vm/core/scenarios/commands.py @@ -359,18 +359,28 @@ def prints(platform: Platform, checks: Checks) -> None: checks.check(f"komorebic {line}", status == 0) -def unknown_monitor(platform: Platform, checks: Checks, clients: list[Client]) -> None: +def unknown_monitor( + platform: Platform, checks: Checks, clients: list[Client], known_failure: str | None = None +) -> None: """Sends the focused window to a monitor that does not exist, and checks that no window is lost. A port may refuse the command or ignore it, but a window that was managed has to stay managed: a daemon that stops managing it leaves it on the screen with nothing tiling it. The platform has fewer than seven displays. + + With `known_failure`, the reason a port is known to lose the window, the checks are recorded as + known failures: they do not fail the suite, and one that holds says the marker can go. The + window is lost then, so the scenario belongs at the end of a suite. """ managed = platform.state().managed() for command in ("send-to-monitor", "move-to-monitor"): platform.komorebic_status(command, "7") platform.wait_for(lambda: unhealthy(platform, clients) is None, timeout=5) - checks.check( - f"after {command} 7, which no display has, the daemon still manages every window", - platform.state().managed() == managed and unhealthy(platform, clients) is None, + description = ( + f"after {command} 7, which no display has, the daemon still manages every window" ) + holds = platform.state().managed() == managed and unhealthy(platform, clients) is None + if known_failure is None: + checks.check(description, holds) + else: + checks.known_failure(description, holds, known_failure) diff --git a/e2e/lib/tests/test_checks.py b/e2e/lib/tests/test_checks.py index dede6580..1b6d7f9b 100644 --- a/e2e/lib/tests/test_checks.py +++ b/e2e/lib/tests/test_checks.py @@ -31,3 +31,18 @@ def test_finish_raises_only_when_something_failed(capsys: pytest.CaptureFixture[ with pytest.raises(AssertionError, match="^1 checks failed: broken$"): checks.finish() assert "passed: 1 failed: 1" in capsys.readouterr().out + + +def test_a_known_failure_is_listed_and_does_not_fail(capsys: pytest.CaptureFixture[str]) -> None: + checks = Checks() + assert checks.known_failure("x", False, "why") is False + checks.finish() + out = capsys.readouterr().out + assert "passed: 0 failed: 0" in out and "known failures: 1" in out + assert checks.known == ["x (why)"] + + +def test_a_known_failure_that_holds_fails_so_the_marker_goes() -> None: + checks = Checks() + checks.known_failure("x", True, "why") + assert checks.failed == 1 and "no longer a known failure" in checks.failures[0] diff --git a/e2e/lib/tests/test_focus_moving.py b/e2e/lib/tests/test_focus_moving.py index 86d09479..40600839 100644 --- a/e2e/lib/tests/test_focus_moving.py +++ b/e2e/lib/tests/test_focus_moving.py @@ -258,3 +258,24 @@ def test_a_daemon_that_ignores_a_monitor_that_does_not_exist_passes() -> None: clients = opened(platform, checks) commands.unknown_monitor(platform, checks, clients) assert checks.failed == 0, checks.failures + + +def test_a_known_failure_does_not_fail_the_suite_until_it_is_fixed() -> None: + class Loses(FakePlatform): + def komorebic_status(self, *args: str) -> tuple[int, str]: + if args[:1] == ("send-to-monitor",): + self.containers[self.focus].clear() + return 0, "" + return super().komorebic_status(*args) + + reason = "the container goes first" + broken, checks = Loses(), Checks() + clients = opened(broken, checks) + commands.unknown_monitor(broken, checks, clients, known_failure=reason) + assert checks.failed == 0, checks.failures + assert len(checks.known) >= 1 + + fixed, checks = FakePlatform(), Checks() + clients = opened(fixed, checks) + commands.unknown_monitor(fixed, checks, clients, known_failure=reason) + assert checks.failed == 2 and checks.known == []