Files
komorebi/e2e/platforms/tart_macos/run.py
T
LGUG2Z 55339cfcb3 test(e2e): the macOS class checks pass, so they are no longer marked
The test window now has a class (its accessibility subrole), and the state has it, so
manage_rules, state_fidelity and rules_class pass against a real guest: the three
known failures of the class difference and the skip of rules_class are removed. A run
with a mark on a check that passes reports it as a failure, so they could not stay.
placement stays skipped, for the initial_window_placement_rules the daemon does not have.
2026-11-06 14:49:36 -08:00

237 lines
9.4 KiB
Python

"""Runs neutral scenarios against a fresh macOS guest.
usage: python -m platforms.tart_macos.run GOLDEN BIN_DIR OUT SCENARIO...
GOLDEN is the name of the golden Tart image, BIN_DIR holds komorebi, komorebic and
komorebi-testwin (built for macOS, see `build-bins.sh`), OUT gets the run's artifacts. Every
scenario gets its own guest, a clone of the golden image. Those that take clients start from the
windows that `tiling` opens.
Every run writes OUT/<scenario>/result.json, naming the scenario, the port, the commit under test
and each check, so that a session can be matched with its run.
With `KOMOREBI_E2E_SCALE=1.5` every wait of a scenario is 1.5 times as long, for a busy host.
With `KOMOREBI_E2E_LOG_LEVEL=debug` the daemon logs at that level.
"""
import importlib
import json
import os
import sys
import time
import traceback
from pathlib import Path
from komorebi_vm.core.suite import shared
from komorebi_vm.runner import RecordedChecks, commit, run_steps, with_kwargs
from .guest import Guest
from .macos_platform import MacosPlatform
BINARIES = ["komorebi", "komorebic", "komorebi-testwin"]
# scenarios that take the windows `tiling` opens
WITH_CLIENTS = {
"focus", "moving", "stacks", "workspaces", "monocle", "closing",
"lifecycle", "layouts", "floating", "minimizing", "padding",
"workspace_moves", "resize_commands", "promoting",
"flipping", "pausing", "tiling_off", "lifecycle_hidden",
"scrolling", "handover", "fullscreen",
}
# what a scenario file has besides `run`, which its suite runs after it
MORE = {
"lifecycle_hidden": ("crash",),
"monocle": ("focus_behaviour",),
}
# scenarios a macOS guest cannot run, and why: they are skipped, not failed
UNSUPPORTED = {
"hotplug": "a macOS guest has one display",
"unknown_monitor": "a macOS guest has one display",
"display_change": "a macOS guest has one display",
"monitors": "a macOS guest has one display",
"focus_follows_mouse": "a macOS guest has no pointer, and the daemon has no focus follows mouse",
"locking": "a macOS guest has no lock screen to test",
"wallpaper": "the wallpaper of a macOS guest is not read yet",
"placement": "the macOS daemon has no initial_window_placement_rules",
"resize": "a macOS guest has no pointer to drag an edge with",
"maximizing": "the macOS komorebic has no toggle-maximize",
"commands": "the sweep of every command is not wired to the macOS command set yet",
}
# what the macOS daemon does differently, which checks of a scenario are about, and the difference
DIFFERENCES = {
"stack": (
"by design: the macOS daemon stacks by raising the focused window over the others, so the "
"other windows of a stack stay on the display at the same place, where Windows and KDE "
"put them away"
),
"fullscreen": (
"not applicable: macOS fullscreen is a Space of its own, which the daemon does not tile, "
"so there is no fullscreen window for it to leave alone; the test window's borderless "
"stand-in is tiled like any window"
),
"margins": (
"the daemon tiles 10 px below the top of the work area the system reports and 12 px above "
"its bottom, so a vertical mirror is a few pixels off"
),
}
# the checks of a scenario that are about one of them, by a part of their description; the others
# of the scenario are checks as any. A mark whose check passes is a failure, to say it can go. The
# marks are of one scenario each: the same words are a check that passes in another.
KNOWN_CHECKS = {
"stacks": {
"exactly one window is put away": "stack",
"after 'cycle-stack next' the other window of the stack is in view and the first is away": "stack",
"cycling back, the first is in view again": "stack",
},
"fullscreen": {
"it does not take the window out of fullscreen": "fullscreen",
"not on any other window": "fullscreen",
},
"layout_flip": {
"BSP, 3 windows, layout_flip Vertical: ... each where the mirror of its place is": "margins",
"BSP, 3 windows, layout_flip HorizontalAndVertical: ... each where the mirror of its place is": "margins",
},
}
KNOWN_CHECKS = {
scenario: {key: DIFFERENCES[name] for key, name in marks.items()}
for scenario, marks in KNOWN_CHECKS.items()
}
class GuestDied(RuntimeError):
"""The guest exited while a scenario was running: not what the daemon did."""
def run_one(golden: str, bin_dir: Path, out: Path, name: str) -> RecordedChecks:
out.mkdir(parents=True, exist_ok=True)
scenario = importlib.import_module(f"komorebi_vm.core.scenarios.{name}")
tiling = importlib.import_module("komorebi_vm.core.scenarios.tiling")
checks = RecordedChecks(KNOWN_CHECKS.get(name, {}))
print(f"== {name}")
began = time.monotonic()
timings: dict[str, float] = {}
def lap(phase: str) -> None:
now = time.monotonic()
timings[phase] = round(now - began - sum(timings.values()), 1)
with Guest(golden, out / "guest") as guest:
lap("guest")
guest.upload(bin_dir, BINARIES)
lap("upload")
platform = MacosPlatform(
guest.agent, out, scale=float(os.environ.get("KOMOREBI_E2E_SCALE", "1"))
)
try:
platform.start_komorebi(log_level=os.environ.get("KOMOREBI_E2E_LOG_LEVEL"))
except Exception:
_diagnose(platform, out)
raise
lap("daemon")
try:
if name in WITH_CLIENTS:
clients = tiling.run(platform, checks)
# `closing` closes the window that was opened last
run_scenario = (
with_kwargs(scenario.run, closing=clients[-1])
if name == "closing"
else scenario.run
)
run_steps(
checks,
shared(run_scenario, platform, checks, clients),
*(
shared(getattr(scenario, more), platform, checks, clients)
for more in MORE.get(name, ())
),
)
else:
run_steps(
checks,
shared(scenario.run, platform, checks),
*(
shared(getattr(scenario, more), platform, checks)
for more in MORE.get(name, ())
),
)
checks.stale_marks()
except Exception as e:
died = guest.died()
if died:
# the guest is gone, so there is nothing to fetch from it
raise GuestDied(died) from e
_diagnose(platform, out)
raise
finally:
lap("scenario")
print(f" seconds: {timings}")
try:
platform.screenshot(f"final-{name}")
except Exception as e: # noqa: BLE001 - a screenshot is for the reader, not a result
print(f" no screenshot: {e}")
(out / "result.json").write_text(json.dumps({
"scenario": name,
"port": "macos",
"commit": commit(),
"passed": checks.passed,
"failed": checks.failed,
"skipped": len(checks.skipped),
"known_failures": len(checks.known),
"start_retries": platform.start_notes,
"seconds": timings,
"steps": checks.steps,
"checks": checks.results,
}, indent=2))
return checks
def _diagnose(platform: MacosPlatform, out: Path) -> None:
"""After a failure: the daemon's output and log, the processes, and a screenshot, into `out`."""
try:
(out / "ps.txt").write_text(platform.agent_exec("ps", "-axo", "pid,etime,command")["stdout"])
platform.screenshot("failure")
except Exception as e: # noqa: BLE001 - diagnosis must not hide the failure it is for
print(f" no diagnosis: {e}")
for name, remote in (("komorebi.out", "/Users/admin/e2e/komorebi.out"),):
try:
(out / name).write_bytes(platform.agent_get_file(remote))
except Exception as e: # noqa: BLE001
print(f" no {name}: {e}")
def main(argv: list[str]) -> int:
golden, bin_dir, out, *names = argv
failed = 0
for name in names:
if name in UNSUPPORTED:
print(f"== {name}: skipped, {UNSUPPORTED[name]}")
continue
try:
try:
checks = run_one(golden, Path(bin_dir), Path(out) / name, name)
except GuestDied as e:
# the guest itself went: not a result about the daemon, so the scenario runs again
print(f"{name}: THE GUEST DIED ({e}); running it again")
checks = run_one(golden, Path(bin_dir), Path(out) / name, name)
except Exception: # noqa: BLE001 - a scenario that cannot run is a failure of its own
traceback.print_exc()
print(f"{name}: did not run, counted as 1 failed")
failed += 1
continue
print(f"{name}: passed {checks.passed} failed {checks.failed}")
if checks.skipped:
print(f"{name}: skipped {len(checks.skipped)}")
if checks.known:
print(f"{name}: known failures {len(checks.known)}")
failed += checks.failed
return 1 if failed else 0
if __name__ == "__main__":
sys.exit(main(sys.argv[1:]))