feat(nix): known failures, with unknown_monitor the first

Checks.known_failure records a check that fails for a reason that is
to be followed up on: the suite does not fail, the totals say how many
known failures there are, and one that holds fails the suite so the
marker goes. unknown_monitor takes the reason, and core-e2e runs it
last, because the window it loses stays lost. BEHAVIOUR.md says so.

docs/dev/BEHAVIOUR.md

@@ -36,7 +36,7 @@ runs, and goes stale when the code changes.
 | `closing` | a closed window leaves the state, and the others take its room | pass | pass |
 | `resize` | dragging the edge two tiles share moves it by what the pointer moved | pass | **fails**, cause found |
 | `commands` | every layout, and about 216 commands, leave the daemon healthy | pass, none skipped | not run, see below |
-| `commands.unknown_monitor` | a window sent to a monitor that does not exist stays managed | **fails** | **fails** |
+| `commands.unknown_monitor` | a window sent to a monitor that does not exist stays managed | **known failure** | **known failure** |
 | `rules` | initial, permanent and ignore rules by title | see below | not run |
 | `hotplug` | an unplugged display takes its windows to the other, a plugged one is a monitor | pass | not run |
 
@@ -54,9 +54,11 @@ present, with the command exiting 0 and printing nothing. On kfde `core-e2e` fai
 check on a real KWin, and the windows after it. kfde's code shows how:
 `WindowManager::move_container_to_monitor` takes the focused container out of its workspace with
 `remove_focused_container()` and only then looks the monitor up, with an error for one that is not
-there, so the container is gone when the error is returned. This is not fixed, on either port. The
-scenario is `commands.unknown_monitor`, which a suite can call; it is not in `core-e2e` while it
-would fail there. The sweep of `commands` is not affected on kfde, which has two outputs, and on
+there, so the container is gone when the error is returned. This is not fixed, on either port, and is a
+**known failure to follow up on later**: `commands.unknown_monitor` takes a `known_failure`
+reason, with which its checks are recorded by `Checks.known_failure`. They do not fail the suite,
+the totals say `known failures: N`, and one that holds fails the suite to say the marker can go.
+`core-e2e` runs it last (section 12c), because the window is lost. The sweep of `commands` is not affected on kfde, which has two outputs, and on
 kfw the sweep now plugs in a second display first.
 
 **`resize` on kfw.** Dragging the edge between two tiles by 30, 60 and 120 pixels moved it by 27,

nix/tests/vm/lib/komorebi_vm/core/checks.py

@@ -16,6 +16,7 @@ class Checks:
         self.failed = 0
         self.failures: list[str] = []
         self.skipped: list[str] = []
+        self.known: list[str] = []
 
     def ok(self, description: str) -> None:
         """Records a check that passed."""
@@ -33,6 +34,19 @@ class Checks:
         self.skipped.append(f"{description} ({reason})")
         print(f"  skip {description}: {reason}")
 
+    def known_failure(self, description: str, condition: bool, reason: str) -> bool:
+        """Records a check that is known to fail, with the reason, to follow up on later.
+
+        The suite does not fail while it does. When it holds, that is a failure, to say that
+        the marker can go.
+        """
+        if condition:
+            self.bad(f"{description}: it holds now, so it is no longer a known failure")
+        else:
+            self.known.append(f"{description} ({reason})")
+            print(f"  known failure {description}: {reason}")
+        return condition
+
     def check(self, description: str, condition: bool | Callable[[], bool]) -> bool:
         """Records `condition` (or what calling it returns) and returns it."""
         holds = condition() if callable(condition) else condition
@@ -51,5 +65,7 @@ class Checks:
         print(f"passed: {self.passed} failed: {self.failed}")
         if self.skipped:
             print(f"skipped: {len(self.skipped)}")
+        if self.known:
+            print(f"known failures: {len(self.known)}")
         if self.failed:
             raise AssertionError(f"{self.failed} checks failed: {'; '.join(self.failures)}")

nix/tests/vm/lib/komorebi_vm/core/scenarios/commands.py

@@ -359,18 +359,28 @@ def prints(platform: Platform, checks: Checks) -> None:
         checks.check(f"komorebic {line}", status == 0)
 
 
-def unknown_monitor(platform: Platform, checks: Checks, clients: list[Client]) -> None:
+def unknown_monitor(
+    platform: Platform, checks: Checks, clients: list[Client], known_failure: str | None = None
+) -> None:
     """Sends the focused window to a monitor that does not exist, and checks that no window is lost.
 
     A port may refuse the command or ignore it, but a window that was managed has to stay managed:
     a daemon that stops managing it leaves it on the screen with nothing tiling it. The platform
     has fewer than seven displays.
+
+    With `known_failure`, the reason a port is known to lose the window, the checks are recorded as
+    known failures: they do not fail the suite, and one that holds says the marker can go. The
+    window is lost then, so the scenario belongs at the end of a suite.
     """
     managed = platform.state().managed()
     for command in ("send-to-monitor", "move-to-monitor"):
         platform.komorebic_status(command, "7")
         platform.wait_for(lambda: unhealthy(platform, clients) is None, timeout=5)
-        checks.check(
-            f"after {command} 7, which no display has, the daemon still manages every window",
-            platform.state().managed() == managed and unhealthy(platform, clients) is None,
+        description = (
+            f"after {command} 7, which no display has, the daemon still manages every window"
         )
+        holds = platform.state().managed() == managed and unhealthy(platform, clients) is None
+        if known_failure is None:
+            checks.check(description, holds)
+        else:
+            checks.known_failure(description, holds, known_failure)

nix/tests/vm/lib/tests/test_checks.py

@@ -31,3 +31,18 @@ def test_finish_raises_only_when_something_failed(capsys: pytest.CaptureFixture[
     with pytest.raises(AssertionError, match="^1 checks failed: broken$"):
         checks.finish()
     assert "passed: 1 failed: 1" in capsys.readouterr().out
+
+
+def test_a_known_failure_is_listed_and_does_not_fail(capsys: pytest.CaptureFixture[str]) -> None:
+    checks = Checks()
+    assert checks.known_failure("x", False, "why") is False
+    checks.finish()
+    out = capsys.readouterr().out
+    assert "passed: 0 failed: 0" in out and "known failures: 1" in out
+    assert checks.known == ["x (why)"]
+
+
+def test_a_known_failure_that_holds_fails_so_the_marker_goes() -> None:
+    checks = Checks()
+    checks.known_failure("x", True, "why")
+    assert checks.failed == 1 and "no longer a known failure" in checks.failures[0]

nix/tests/vm/lib/tests/test_focus_moving.py

@@ -258,3 +258,24 @@ def test_a_daemon_that_ignores_a_monitor_that_does_not_exist_passes() -> None:
     clients = opened(platform, checks)
     commands.unknown_monitor(platform, checks, clients)
     assert checks.failed == 0, checks.failures
+
+
+def test_a_known_failure_does_not_fail_the_suite_until_it_is_fixed() -> None:
+    class Loses(FakePlatform):
+        def komorebic_status(self, *args: str) -> tuple[int, str]:
+            if args[:1] == ("send-to-monitor",):
+                self.containers[self.focus].clear()
+                return 0, ""
+            return super().komorebic_status(*args)
+
+    reason = "the container goes first"
+    broken, checks = Loses(), Checks()
+    clients = opened(broken, checks)
+    commands.unknown_monitor(broken, checks, clients, known_failure=reason)
+    assert checks.failed == 0, checks.failures
+    assert len(checks.known) >= 1
+
+    fixed, checks = FakePlatform(), Checks()
+    clients = opened(fixed, checks)
+    commands.unknown_monitor(fixed, checks, clients, known_failure=reason)
+    assert checks.failed == 2 and checks.known == []

nix/tests/vm/suites/core-e2e/test.py

@@ -16,6 +16,7 @@ from komorebi_vm import Checks, Kde, Rect, daemon
 from komorebi_vm.core.platform import Client
 from komorebi_vm.core.scenarios import (
     closing,
+    commands,
     focus,
     lifecycle,
     monocle,
@@ -467,6 +468,16 @@ platform.close(Client("after-the-restart"))
 platform.close(Client("beside"))
 checks.check("one window is left", kde.wait_for(lambda: kde.managed_all() == 1, timeout=20))
 
+print("== 12c. a window sent to a monitor that does not exist")
+# KNOWN FAILURE, to follow up on: move_container_to_monitor takes the container out before it
+# looks the monitor up. See docs/dev/BEHAVIOUR.md. The window is lost, so this stays last.
+commands.unknown_monitor(
+    platform,
+    checks,
+    [Client("afterwards")],
+    known_failure="the container is removed before the monitor is looked up, on both ports",
+)
+
 print("== 13. komorebic stop")
 kde.komorebic("stop")
 checks.check("the daemon exited", kde.wait_for(lambda: not kde.komorebi_running()))