diff --git a/.github/workflows/recorder.yaml b/.github/workflows/recorder.yaml new file mode 100644 index 00000000..5dfc0e2d --- /dev/null +++ b/.github/workflows/recorder.yaml @@ -0,0 +1,241 @@ +# Adapted from komorebi's windows workflow, with a macOS job added. One workflow builds both +# platforms and one job publishes what they built. + +name: ci + +on: + pull_request: + branches: + - "*" + push: + branches: + - master + - feature/* + - hotfix/* + tags: + - v* + schedule: + - cron: "30 0 * * 0" # Every Sunday at 00:30 UTC + workflow_dispatch: + +jobs: + cargo-deny: + if: github.repository == 'LGUG2Z/komorebi-recorder' + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v6 + with: + fetch-depth: 0 + - uses: EmbarkStudios/cargo-deny-action@v2 + + # `nix flake check` includes the Windows clippy check, which type-checks every `cfg(windows)` line + macos: + if: github.repository == 'LGUG2Z/komorebi-recorder' + runs-on: self-hosted + permissions: write-all + env: + GH_TOKEN: ${{ github.token }} + steps: + - uses: actions/checkout@v6 + with: + fetch-depth: 0 + - run: nix flake check + - run: nix build + - uses: actions/upload-artifact@v7 + with: + name: komorebi-recorder-aarch64-apple-darwin-${{ github.sha }} + path: result/bin/komorebi-recorder + retention-days: 14 + + # the Linux host (KDE Plasma): `nix flake check` builds it, clippy and the tests included (those of + # the screen need a Plasma session and are not among them), and type-checks the Windows code too + linux: + if: github.repository == 'LGUG2Z/komorebi-recorder' + runs-on: ubuntu-latest + env: + GH_TOKEN: ${{ github.token }} + steps: + - uses: actions/checkout@v6 + with: + fetch-depth: 0 + - uses: DeterminateSystems/nix-installer-action@main + - run: nix flake check + - run: nix build + - uses: actions/upload-artifact@v7 + with: + name: komorebi-recorder-x86_64-unknown-linux-gnu-${{ github.sha }} + path: result/bin/ + retention-days: 14 + + windows: + if: github.repository == 'LGUG2Z/komorebi-recorder' + strategy: + fail-fast: true + matrix: + platform: + - os-name: Windows-x86_64 + runs-on: windows-latest + target: x86_64-pc-windows-msvc + - os-name: Windows-aarch64 + runs-on: windows-latest + target: aarch64-pc-windows-msvc + runs-on: ${{ matrix.platform.runs-on }} + permissions: write-all + env: + RUSTFLAGS: -Ctarget-feature=+crt-static -Dwarnings + GH_TOKEN: ${{ github.token }} + steps: + - uses: actions/checkout@v6 + with: + fetch-depth: 0 + # There is no nix here, so the komorebi sources the recorder indexes (and its tests read) are + # checked out at the revisions flake.lock pins, which stays the single place they are pinned + - name: Check out the komorebi sources at the revisions flake.lock pins + shell: bash + run: | + for input in komorebi-mac:LGUG2Z/komorebi-for-mac komorebi-windows:LGUG2Z/komorebi; do + name="${input%%:*}" + repository="${input#*:}" + rev="$(jq -r ".nodes[\"$name\"].locked.rev" flake.lock)" + directory="$RUNNER_TEMP/$name" + git init --quiet "$directory" + git -C "$directory" fetch --quiet --depth 100 "https://github.com/$repository" "$rev" + git -C "$directory" checkout --quiet FETCH_HEAD + variable="$(echo "$name" | sed -e 's/^komorebi-mac$/KOMOREBI_MAC/' -e 's/^komorebi-windows$/KOMOREBI_WINDOWS/')" + echo "${variable}_SRC=$(cygpath -m "$directory")" >> "$GITHUB_ENV" + echo "${variable}_COMMIT=$rev" >> "$GITHUB_ENV" + done + - run: rustup toolchain install stable --profile minimal + - run: rustup component add --toolchain stable-x86_64-pc-windows-msvc clippy + - run: rustup toolchain install nightly --allow-downgrade -c rustfmt + - uses: Swatinem/rust-cache@v2 + with: + cache-on-failure: "true" + cache-all-crates: "true" + key: ${{ matrix.platform.target }} + # the checks run on the x86_64 host, once + - if: matrix.platform.target == 'x86_64-pc-windows-msvc' + run: cargo +nightly fmt --check + - if: matrix.platform.target == 'x86_64-pc-windows-msvc' + run: cargo clippy --workspace --all-targets --locked -- -D warnings + # Informational until it has been seen green on Windows: the tests were written on a Mac and + # nothing has run them here. Make it blocking by removing `continue-on-error`. + - if: matrix.platform.target == 'x86_64-pc-windows-msvc' + run: cargo test --workspace --locked + continue-on-error: true + - uses: houseabsolute/actions-rust-cross@v1 + with: + command: "build" + target: ${{ matrix.platform.target }} + args: "--locked --release" + - shell: pwsh + run: | + cargo install cargo-wix + cargo wix --no-build -p komorebi-recorder --nocapture -I .\wix\main.wxs --target ${{ matrix.platform.target }} + - uses: actions/upload-artifact@v7 + with: + name: komorebi-recorder-${{ matrix.platform.target }}-${{ github.sha }} + path: | + target/${{ matrix.platform.target }}/release/*.exe + target/${{ matrix.platform.target }}/release/*.pdb + target/wix/komorebi-recorder-*.msi + retention-days: 14 + + # Packages what the two builds made and publishes it. What kind of publication depends on the + # event: a push of a `v*` tag releases; a scheduled or manual run on master replaces the rolling + # `nightly` prerelease; any other push to master makes a draft release to look at. + publish: + if: github.repository == 'LGUG2Z/komorebi-recorder' && (github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/tags/v')) + needs: [macos, windows] + runs-on: self-hosted + permissions: write-all + env: + GH_TOKEN: ${{ github.token }} + steps: + - uses: actions/checkout@v6 + with: + fetch-depth: 0 + - name: Decide what is being published + shell: bash + run: | + if [[ "$GITHUB_REF" == refs/tags/v* ]]; then + echo "MODE=release" >> "$GITHUB_ENV" + echo "VERSION=${GITHUB_REF_NAME#v}" >> "$GITHUB_ENV" + elif [[ "$GITHUB_EVENT_NAME" == schedule || "$GITHUB_EVENT_NAME" == workflow_dispatch ]]; then + echo "MODE=nightly" >> "$GITHUB_ENV" + echo "VERSION=nightly" >> "$GITHUB_ENV" + else + echo "MODE=draft" >> "$GITHUB_ENV" + echo "VERSION=$(sed -n 's/^version = "\(.*\)"$/\1/p' Cargo.toml | head -n 1)" >> "$GITHUB_ENV" + fi + - uses: actions/download-artifact@v8 + - name: Package + shell: bash + run: | + mkdir dist + # artifacts do not keep the executable bit + chmod +x "komorebi-recorder-aarch64-apple-darwin-${GITHUB_SHA}/komorebi-recorder" + zip -j "dist/komorebi-recorder-${VERSION}-aarch64-apple-darwin.zip" \ + "komorebi-recorder-aarch64-apple-darwin-${GITHUB_SHA}/komorebi-recorder" + for triple in x86_64-pc-windows-msvc aarch64-pc-windows-msvc; do + arch="${triple%%-*}" + zip -j "dist/komorebi-recorder-${VERSION}-${triple}.zip" \ + "komorebi-recorder-${triple}-${GITHUB_SHA}/${triple}/release/"*.exe + cp "komorebi-recorder-${triple}-${GITHUB_SHA}/wix/"*"-${arch}.msi" \ + "dist/komorebi-recorder-${VERSION}-${arch}.msi" + done + (cd dist && shasum -a 256 -- * > checksums.txt) + - name: Write the release notes + shell: bash + run: | + if ! type kokai >/dev/null; then cargo install --locked kokai --force; fi + case "$MODE" in + nightly) + git tag -d nightly || true + git tag nightly + ref=nightly ;; + draft) ref="$GITHUB_REF_NAME" ;; + release) ref="$GITHUB_REF_NAME" ;; + esac + kokai release --no-emoji --add-links github:commits,issues --ref "$ref" > "$RUNNER_TEMP/release-notes.md" + - name: Publish + shell: bash + run: | + case "$MODE" in + nightly) + gh release delete nightly --yes || true + git push origin :nightly || true + gh release create nightly \ + --target "$GITHUB_SHA" \ + --prerelease \ + --title "komorebi-recorder nightly (${GITHUB_SHA})" \ + --notes-file "$RUNNER_TEMP/release-notes.md" \ + dist/* ;; + draft) + # a draft is not found by its tag, so the previous one is found in the list + gh api "repos/${GITHUB_REPOSITORY}/releases" --paginate \ + --jq '.[] | select(.draft and .tag_name == "draft") | .id' | + while read -r id; do gh api -X DELETE "repos/${GITHUB_REPOSITORY}/releases/${id}"; done + gh release create draft \ + --draft \ + --target "$GITHUB_SHA" \ + --title "komorebi-recorder ${VERSION} (draft, ${GITHUB_SHA})" \ + --notes-file "$RUNNER_TEMP/release-notes.md" \ + dist/* ;; + release) + gh release create "$GITHUB_REF_NAME" \ + --verify-tag \ + --title "komorebi-recorder ${VERSION}" \ + --notes-file "$RUNNER_TEMP/release-notes.md" \ + dist/* ;; + esac + + winget: + if: github.repository == 'LGUG2Z/komorebi-recorder' && startsWith(github.ref, 'refs/tags/v') + needs: publish + runs-on: ubuntu-latest + steps: + - uses: vedantmgoyal2009/winget-releaser@main + with: + identifier: LGUG2Z.komorebi-recorder + token: ${{ secrets.WINGET_TOKEN }} diff --git a/docs/recorder/ARCHITECTURE.md b/docs/recorder/ARCHITECTURE.md new file mode 100644 index 00000000..8e3fa4ec --- /dev/null +++ b/docs/recorder/ARCHITECTURE.md @@ -0,0 +1,114 @@ +# Architecture + +`komorebi-recorder` connects three things that do not know about each other: komorebi, which says what it is doing in +a log and in events on a socket; the operating system, which says what it is doing in its own trace and shows it on a +screen; and the person or agent who has to work out why they disagreed. It records all of them on one clock into a +session directory, and reads that directory back. + +Two things about it vary independently, and everything below is arranged so that neither leaks into the other. + +| axis | what varies | where the variation lives | what a session records | +|---|---|---|---| +| host | the OS the recorder runs on: macOS or Windows (a third `unsupported` host builds anywhere) | [`platform`](../komorebi-recorder/src/platform/mod.rs), selected by `cfg` | `host` in the manifest | +| flavor | the komorebi being recorded: komorebi for Mac or komorebi for Windows | `Rules` in [`analysis::flavor`](../komorebi-recorder/src/analysis/flavor/mod.rs), selected by data | `komorebi.flavor` in the manifest | + +They are independent because a session recorded on one host is investigated on whichever machine the investigator has. +A session recorded on Windows is read on a Mac, and the other way round, so reading never asks which host it is +running on: it asks the session. + +## The master clock + +Everything in a session is stamped on one clock: the host's monotonic clock, in nanoseconds. That is +`mach_absolute_time` on macOS and `QueryPerformanceCounter` on Windows. Video frames and the OS trace are already on it. +komorebi's log lines carry UTC wall-clock time, so they are mapped onto it: the recorder samples a paired reading of +both clocks (an `Anchor`) at the start, every few seconds and at every sleep, and a `ClockMap` interpolates between +them. A `JumpDetector` notices sleep and clock steps by comparing how far each clock moved between two observations, +and the anchors on either side of the jump are what make the mapping right on both sides of it. + +Three things are worth knowing before reading a session: + +- The field names ending in `_mach_ns` keep their historical spelling. They hold master-clock nanoseconds on whichever + host recorded the session; the manifest's `host.clock` says which clock that was (`mach`, `qpc` or `std`). A name that + changed would stop older sessions being readable by newer builds and the reverse. +- The master clock is not the wall clock. On macOS it pauses while the machine sleeps, so video time skips a sleep + instead of showing a hole. Whether the performance counter counts a suspend on Windows is measured, not assumed. + (`doctor --probe clock` measures it.) +- A timestamp is a plain `u64` of nanoseconds. Nothing in the type system says which clock it is on: that is held by + convention, and by the tests that place records at known times. + +## The platform facade + +The rest of the crate never names a system API. [`platform`](../komorebi-recorder/src/platform/mod.rs) is a facade of +small modules (`clock`, `shell` and, as the recorder grows, `capture`, `power`, `session`, `trace`, `process`, `apps` +and `window`), and `cfg` selects the directory behind it: `platform/macos/`, `platform/win/` or `platform/unsupported.rs`. +Each host provides the same names with the same signatures. A missing or mismatched function is a compile error on the +host being built, and on no other, so the Windows host is built from a Mac by `just check-windows` and by CI, and +`unsupported` (which exists so that the commands that only read a session directory work on a Linux agent box) is built +only on a third OS. + +Where one host has no analogue of something, the facade function exists on both and one answers "nothing": `trace::start` +on the `unsupported` host returns no taps and a line saying so, and `power::start` returns no watcher. + +## Flavors and rules + +Which komorebi produced a session decides what its events mean (a process id and an accessibility notification on a Mac, +a window handle and a WinEvent on Windows), how it hides a window (by moving it to a corner on a Mac, by cloaking it in +place on Windows) and how its log marks where a handler starts and ends. `Flavor` names the two and is recorded in the +manifest. `Rules` is the trait that holds what each knows, and `Flavor::rules` is where a name becomes rules. Code that +reads komorebi's output asks `Rules` and does not match on `Flavor`; only convention holds that, and it has exceptions: +the index of komorebi's source that log lines are mapped to, and the places the log and a checkout of the source are +looked for on disk, are keyed by `Flavor` directly. + +The contract is a counting rule: a flavor implements every method of `Rules` that has no default, and the compiler +refuses an implementor that misses one. A method with a default is a question most flavors answer the same way, and the +doc of each says what the default is. `Rules::verified` is how a flavor says that its rules were written from komorebi's +source and have not been checked against a recording of it: an analysis that reads such a session says so. + +## The session + +A session is a directory. Its manifest, `session.json`, is the one file everything else is read through: the host and the +flavor, the clock anchors, the komorebi processes that were seen and how each was logging, the byte ranges of komorebi's +log that were recorded and the applications that were traced. It is written to a temporary file and renamed, so a reader +or a crash never sees half of one, and its `schema_version` is checked on every read: a manifest written by a newer +build is refused with an error that says so, rather than read wrongly. + +The files beside it are the raw records, and they are written as the data arrives and never rewritten: komorebi's log +(`komorebi.log`, the exact byte range recorded), its events (`events.jsonl`, with full `State` snapshots in `states/` +only when they changed), and what happened to the recording itself (`recorder.jsonl`). +The OS trace is under `oslog/`, one file per stream. +Each display's video is `video-[-].mp4` with a `frames-[-].csv` beside it holding every frame's timestamp, +one segment to a file: a display that is unplugged and returns starts a new segment instead of extending the old one. +`timeline.jsonl` is the only derived file: a time-sorted index of every record that points into the raw files by byte +range. Anything that happens to the recorder leaves the raw files readable, and the index is rebuilt from them with +`reindex`. The file formats are specified on the `session` and `timeline` modules. + +## Recording + +The recorder starts its sources in an order that makes the sources that are not video a superset of the video's span +(the clock anchor, the OS trace streams, the subscription to komorebi and its first `State`, komorebi's log offset, and +last the video), and stops them in the reverse order. +While it runs, the machine is docked, undocked, put to sleep, woken with the lid opened and locked, and none of that may +lose what was already captured. A tracker turns what the OS says (or, on a Mac, what the two clocks disagree about) into +reactions: a display that is gone has its video finalized at once, one that returns starts a new segment, and every +stream is started afresh after a wake. The capture backend is a trait, so that these policies are tested with a scripted +backend that docks and undocks without hardware. + +## Investigating + +Reading a session never needs a window: `analysis` answers each question as a plain serializable value, and the command +line and the viewer present the same answers. Every command takes `--json`, which prints an envelope with the data, its +schema, warnings, whether it was truncated and a cursor, and failures exit with a code that says what kind they were. + +## Rules and the layer that enforces them + +| Rule | Enforced by | +|---|---| +| Every host provides the same functions with the same signatures | the compiler, for the host being built; `just check-windows` and CI build the Windows host from a Mac; nothing builds `unsupported` | +| A timestamp in a session is on the master clock | convention, and the tests that place records at known times: the type is a plain `u64` | +| An implementor of `Rules` answers every question that has no default | the type system: a missing method is E0046, proved by a `compile_fail` example on `Rules` | +| Code that reads komorebi's output asks `Rules` and does not match on `Flavor` | convention only | +| Rules that were not checked against a recording of komorebi say so | the framework path: an analysis that reads such a session adds a warning | +| A running recorder can be told from one that died | the OS: an exclusive lock on `recorder.lock`, held for as long as the recorder runs and dropped by the OS when the process ends however it ends | +| A manifest from a newer build is not read | a runtime error when the manifest is read, covered by `rejects_manifests_from_the_future` | +| A manifest is never half-written | write to a temporary file and rename | +| The raw files are never rewritten | convention: the index only references them, and `reindex` rebuilds the index from them | diff --git a/docs/recorder/CONTRIBUTING.md b/docs/recorder/CONTRIBUTING.md new file mode 100644 index 00000000..cd30ebcb --- /dev/null +++ b/docs/recorder/CONTRIBUTING.md @@ -0,0 +1,81 @@ +# Contributing to komorebi-recorder + +The project is a collection of contributions from both the project leaders and +community members. There are many ways to contribute, this can include content +in the project repositories, as well as contributing in public and private +conversation, assisting users, writing blog posts, and many other ways. + +## How contributions are made + +Contributions to the project primarily happen in the project source +repositories, but may also occur in other places, such as discussion forums and +public and private discourse. + +## Contributing content to the Project + +In order for the project leaders to manage sustained progress toward the +project goals and maintain project velocity, focus and quality, the project may +adjust the license terms over time. + +Content contributed to the project must therefore be provided under +sufficiently liberal terms to allow these operations to proceed unimpeded. As +such contributions are accepted with the following understanding: + +* Contributed content is licensed under the terms of the 0-BSD license +* Contributors accept the terms of the project license at the time of + contribution + +By making a contribution, you accept both the current project license terms, +and that all contributions that you have made are provided under the terms of +the 0-BSD license. + +## Zero-Clause BSD + +``` +Permission to use, copy, modify, and/or distribute this software for +any purpose with or without fee is hereby granted. + +THE SOFTWARE IS PROVIDED “AS IS” AND THE AUTHOR DISCLAIMS ALL +WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES +OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE +FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY +DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN +AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT +OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +``` + +## Development environment + +The project is built with Nix. `nix develop` (or direnv, whose `.envrc` is `use flake`) provides the +Rust toolchain with nightly `rustfmt`, `cargo-deny`, `cargo-nextest`, `ffmpeg`, `jq`, `just`, +`resvg` and `ripgrep`, and sets `KOMOREBI_MAC_SRC` and `KOMOREBI_WINDOWS_SRC` (with `_COMMIT`) to the komorebi +sources pinned in `flake.lock`. The tests that read komorebi's source need them, so run `cargo` inside +the shell. + +The Windows code is type-checked from a Mac with `nix develop .#windows`, whose target is +`x86_64-pc-windows-msvc`. Nothing there runs it: a change to a `cfg(windows)` line is first run by the +`windows` job in CI. +On a Windows machine, the steps in [docs/dev/windows-checkpoints.md](docs/dev/windows-checkpoints.md) +say what to run and what to look for. + +The icon is `komorebi-recorder/assets/watermelon.svg`. After changing it, run `just icon` to render the PNG that +the windows set and the ICO that Windows embeds, and commit the three together. + +## Checks + +Before pushing, run + +```sh +just check +``` + +which runs formatting, clippy and the tests for macOS, clippy for Windows, rustdoc with its warnings as +errors for both hosts, and `cargo deny`. `just --list` shows the other recipes. + +## Commits + +Commit messages follow [Conventional Commits](https://www.conventionalcommits.org/), as the pull request +template asks: the release notes are generated from them. + +A change that alters what the documentation says (the rustdoc, the README, `docs/`) updates the +documentation in the same commit, so that the documentation never describes code that is not there. diff --git a/docs/recorder/DESIGN.md b/docs/recorder/DESIGN.md new file mode 100644 index 00000000..5cf27fe8 --- /dev/null +++ b/docs/recorder/DESIGN.md @@ -0,0 +1,122 @@ +# Design + +This document records why the recorder is built the way it is. It is organized as questions, because each answer is a +decision that could have gone another way. The facts in it were measured or read from the code, and where a reason is a +judgment it says so. It is written in the first person only where the decision was the owner's. + +## Why record the screen, the log, the events and the OS trace together? + +Most of the komorebi bugs I see show up when a laptop is docked or undocked, or wakes when the screen is opened. To +reproduce one, I had to watch the screen, hand-grep komorebi's log and export the system log from Console, and there was +no way to say: at 00:12.5 in the video, what was komorebi doing, and what was the OS doing? The recorder puts the four on +one clock so that the question has an answer, and so that the answer also works the other way, from a log line to the +moment in the video. + +OS-level tracing is in scope because the interesting seam is between komorebi and the system, and between the system and +the application being tiled; komorebi's own log only shows one side of each. + +## Why one clock, and why that one? + +Every record is stamped on the host's monotonic clock (`mach_absolute_time` on macOS, `QueryPerformanceCounter` on +Windows) because the two sources that cannot be re-stamped are already on it: ScreenCaptureKit frame timestamps and the +unified log's `machTimestamp` on a Mac, and Windows.Graphics.Capture frame times on Windows. komorebi's log lines carry +UTC wall-clock time, so they are mapped onto the master clock with paired readings of both clocks (anchors) sampled while +recording, interpolated piecewise so that a step or a sleep between two anchors does not smear the mapping. + +The alignment is as good as that allows: video to OS trace within about one frame, and video to komorebi's log within +about one frame plus the error of the wall-clock anchor, tens of microseconds. + +## Why are fields called `*_mach_ns` on Windows? + +They hold nanoseconds on the master clock, whichever clock that was. The spelling is historical and was kept: renaming +them would make a session written by one build unreadable by another. The manifest's `host.clock` says which clock a +session was on. + +## Why does the recorder never change komorebi's log level or lifecycle? + +That was decided at the start: the recorder only observes. It warns when komorebi is logging at a level that leaves out +most of what the window manager decided, and says how to restart it, but it never restarts komorebi or changes its +settings. (The reason for the decision is not written down anywhere the design could be checked against; this entry +needs the owner's words.) + +## Why does the accept loop never block, and why is an event stamped when its handler finished? + +komorebi connects to each subscriber synchronously and sends a notification while it holds its own locks, so a recorder +that was slow to read would stall komorebi. The accept thread stamps the arrival, reads, and hands the event to a channel; +a writer thread does everything else. For the same reason the per-event window probe (`--probe-windows`) runs on the +writer thread and never on the accept thread. + +komorebi sends the notification after it has handled the event, so the stamp is when the handler finished. The start of +the handler is the `processing event` line in komorebi's log, which is why `trace` pairs the two. + +## Why ScreenCaptureKit through `objc2` and not the `screencapturekit` crate? + +The `screencapturekit` crate needs a Swift bridge at build time, which complicates the nix build. `objc2-screen-capture-kit` +and its siblings exist at the versions the rest of komorebi already uses, so there is no Swift and no Xcode dependency. + +## Why fragmented mp4, and why re-submit the last frame? + +A recorder that is killed, or a machine that loses power, must not lose the recording, and a plain mp4 is unreadable +until it is finalized. Fragmented mp4 plays up to the last fragment. Two measured facts decided the rest: ScreenCaptureKit +only emits frames when the screen changes, and fragments are cut on media time, so a static display never flushed a +fragment; the capture therefore re-submits the last frame to keep it moving. And the duration `ffprobe` reports for a +truncated fragmented file under-reads, so recovery asks AVFoundation instead. + +## Why a new file for every segment? + +Docking and undocking, sleep and lock change what can be recorded, and I want the recording to carry on through all of +them without losing what was already captured. A display that disappears has its video finalized on the spot, so the file +is complete and playable whatever happens next, and a display that returns starts a new segment in a new file. Every +display's segments are listed in the manifest with their time spans, so a gap is a fact the session states rather than a +hole. + +## Why raw files plus an index, and not one file or a database? + +The raw files are written as the data arrives and are never duplicated or rewritten; `timeline.jsonl` only references byte +ranges in them, so the streams stay separate and can be reconciled or re-indexed later. The index can be rebuilt from the +raw files at any time (`reindex`), including after a crash and after a hand edit. + +## Why does reading a session never ask which host it is on? + +A session recorded on Windows is investigated on a Mac, and the other way round. The host and the komorebi flavor are +recorded in the manifest, and everything that reads a session asks the manifest. + +## Why can a bug only be marked in the viewer? + +The Record tab has no button for marking a bug, and a session's tab has one. I found it confusing to have two places to do +the same thing, so a bug is marked in a session's tab only. `komorebi-recorder annotate` writes the same annotations for a +script or an agent that has no window. + +## Why is there one window? + +Recording and reviewing used to be two windows, and the recorder's buttons for opening a recording started the other one. +Now the recorder is the pinned first tab of the one window and each recording is a tab after it, so there is one thing to +keep out of the video and out of komorebi's layout. It starts small, always on top, with only the Record tab, which is what +the old recorder window was for, and grows when a recording is opened. + +## Why is the window untitled? + +komorebi does not manage windows with an empty title, so an untitled window is never tiled: opening the recorder +does not move the layout of the machine whose bug is being recorded. + +## Why is ffmpeg not part of the program? + +ffmpeg is used only to extract frames (`frames`) and to make test videos, and never for capture. It is in the dev shell and +not in the packaged binary. + +## Why is a komorebi source from a different commit "skewed" by files and not by commit? + +The recorder's own commits advance the repository's head without touching komorebi, so comparing commits calls almost +everything skewed. The files that differ between the commit a session recorded and the commit indexed are what say whether +a line of the log can still be traced to the statement that wrote it. + +## Why is the hidden-window check skipped on Windows? + +komorebi for Windows hides a window by cloaking it where it is, so a window's rectangle says nothing about whether it is +hidden. The check that flags a window parked off screen on the showing workspace holds only where komorebi hides windows +by moving them, so it is skipped on Windows, and `--probe-windows` reads the real state from Windows instead. + +## Why do the Windows rules say they are unverified? + +They were written from komorebi for Windows' source. Until a recording of it has been read, an analysis that uses them says +so; `Rules::verified` is the flag, and it is flipped once the first checks on a Windows machine confirm the rules. diff --git a/docs/recorder/PLATFORM.md b/docs/recorder/PLATFORM.md new file mode 100644 index 00000000..7aa8efbf --- /dev/null +++ b/docs/recorder/PLATFORM.md @@ -0,0 +1,200 @@ +# Platform support + +This document gathers, in one place, what differs by operating system. Most of the details are written down on the items +that implement them; here the same questions are answered once per host, so that someone who has to run the recorder on +a machine, or change it for one, does not have to read the code to find out. + +The recorder is built for macOS, Windows and Linux (KDE Plasma). A fourth host, `unsupported`, builds on anything else so +that the commands that only read a session directory work there; recording says that it cannot. + +## Status + +| host | built | run | +|---|---|---| +| macOS (Apple silicon) | yes | yes: written and run on macOS 26 | +| Windows x86_64 | yes, natively on `windows-latest` and cross-compiled from a Mac | no: type-checked and cross-built only | +| Windows ARM64 | yes, cross-built on an x86_64 runner | no | +| Linux (KDE Plasma, x86_64) | yes, natively | decoding, the komorebi side of recording and the OS trace were run, with komorebi for KDE's in-memory desktop; the screencast was not: see [kde-checkpoints.md](dev/kde-checkpoints.md) | +| anything else | the `unsupported` host builds | nothing records | + +Nothing that touches a real Windows desktop has been seen running: the Windows host was written from the Windows API +documentation and komorebi for Windows' source, and every claim below about Windows says what the code does, not what +was observed. +[windows-checkpoints.md](dev/windows-checkpoints.md) lists the checks that turn each into an observation. + +## Vocabulary + +- **Host**: the OS the recorder runs on. It decides how the screen is captured, which clock is read, how the OS says it + is going to sleep and what the OS trace is made of. +- **Flavor**: which komorebi is being recorded. There is one per OS, and a recorder watches the one on its own host, but + a session records its flavor, so a session from either can be read on either host. +- **Master clock**: the host's monotonic clock, in nanoseconds, on which every record in a session is stamped. +- **Session**: the directory a recording is written to. +- **Segment**: one contiguous stretch of one display's video, a file of its own. A display that is unplugged and plugged + in again has two. + +## The same questions, per host + +### The master clock + +| | macOS | Windows | unsupported | +|---|---|---|---| +| clock | `mach_absolute_time`, scaled by `mach_timebase_info` | `QueryPerformanceCounter`, scaled by `QueryPerformanceFrequency` (10 MHz when Windows will not say) | the time since the first reading in the process, plus one second so that no reading is zero | +| `host.clock` in the manifest | `mach` | `qpc` | `std` | +| counts the time the machine sleeps | no | not known: measured, not assumed | no | +| other clocks read beside it | `mach_continuous_time`, which counts sleep | `QueryInterruptTime`, which counts sleep, and `QueryUnbiasedInterruptTimePrecise`, which does not | none | + +Video frame times and the OS trace are on the master clock already on both hosts (ScreenCaptureKit and the unified log's +`machTimestamp` on a Mac; Windows.Graphics.Capture on Windows). The field names ending in `_mach_ns` are spelled that way +on every host: see [the architecture](ARCHITECTURE.md#the-master-clock). + +### Noticing sleep, wake, lock and display changes + +| | macOS | Windows | +|---|---|---| +| a sleep | found afterwards: the master clock pauses while the wall clock does not, which the recording loop sees by comparing the two many times a second | heard as it happens, from a hidden window on a thread of its own that receives suspend and resume; the lid, the displays' power state and session lock are registered for | +| the screen locked | polled: CoreGraphics' session dictionary | polled: being refused the input desktop means a lock screen or a UAC prompt, and the screen cannot be captured then either way | +| displays added, removed, resized | polled: `CGGetActiveDisplayList` and `CGDisplayIsAsleep`, which are cheap and need no permission | polled, and heard as a display-change message | + +Windows is told when the lock happens as well as polling for it. What is done about each of these, on both hosts, is the +same: a display that is gone has its video finalized at once, a display that returns starts a new segment, and every +stream is started afresh after a wake instead of trusting the old one. + +### Capturing the screen + +| | macOS | Windows | +|---|---|---| +| API | ScreenCaptureKit, through `objc2` bindings, and AVFoundation's `AVAssetWriter` | Windows.Graphics.Capture (Windows 10 version 1803 or later), and a Media Foundation MP4 sink | +| permission | Screen Recording, granted to the terminal or app that launches the binary | none | +| default codec | HEVC | H.264 | +| other codec | H.264 (`--codec h264`) | HEVC is refused with an error: Media Foundation's MP4 sink does not write it | +| what is captured | each display, excluding the recorder's own processes by name | each monitor whole, so the program's windows ask to be left out (see below) | +| a still screen | a healthy stream still calls back about 30 times a second, so silence for four seconds means a stalled stream | Windows sends nothing while the picture does not change, so silence says nothing there | + +The video of a display is written as fragmented MP4 on both hosts, in segments, so that the part written before a crash +can still be played. Because ScreenCaptureKit only emits frames when the screen changes, the macOS capture re-submits the +last frame so that a static display still flushes fragments. + +### The OS trace + +| | macOS | Windows | +|---|---|---| +| source | the unified log, through `log stream --style ndjson` child processes | WinEvents for every top-level window, through a hook, and the System Event Log | +| files | `oslog/komorebi.ndjson`, `oslog/system.ndjson` and one `oslog/apps/-.ndjson` per managed application | `oslog/system.ndjson` (the WinEvents, repeats folded) and `oslog/eventlog.ndjson` (warnings, errors and the display, power and GPU providers) | +| permissions | none: no `sudo` and no change to SIP | none | +| redaction | `` values stay redacted | not applicable | + +A host with no OS trace says so in the recording instead of failing to start streams that do not exist. + +### Locking a session + +A recording holds an exclusive lock on `recorder.lock` in its session for as long as it runs, and writes its process id +to `recorder.pid`. The OS drops the lock when the process dies, however it dies, so a lock that is held means a live +recorder and one that is free means there is none. Windows file locks are mandatory, which is why the pid is not kept in +the locked file. + +### komorebi's log and how the recorder talks to it + +| | komorebi for Mac | komorebi for Windows | +|---|---|---| +| data directory | `~/Library/Application Support/komorebi` | `%LOCALAPPDATA%\komorebi` | +| log | `komorebi_plaintext.log.` in the data directory | `%TEMP%\komorebi_plaintext.log.` | +| command and subscriber sockets | unix domain sockets in the data directory | unix domain sockets in the data directory, through `uds_windows` | +| `--log-level debug` | `komorebi --log-level debug` | `komorebi.exe --log-level debug`; `RUST_LOG=debug` also prints every dependency, so name the crate: `RUST_LOG=komorebi=debug` | + +The wire protocol is the same on both. The recorder's own binary does not depend on either komorebi's crates. + +### Decoding the recorded video + +macOS decodes with AVFoundation's `AVAssetReader`, Windows with Media Foundation's Source Reader, and the `unsupported` +host with nothing: it says so when asked to open a file. Both real hosts are asked for BGRA frames at a chosen size, so +that a contact sheet never decodes more pixels than it shows. + +### The window track + +`record` samples every top-level window (on by default; `--window-track=HZ` picks the rate, `--no-window-track` turns it off) into `windows.jsonl` (see +`src/sources/windowtrack.rs`), listed by `platform::window::list`. + +| | macOS | Windows | +|---|---|---| +| source | `CGWindowListCopyWindowInfo(OptionAll)`, layer 0 windows | `EnumWindows` once per tick, then cheap reads of each window | +| `id` | the window server's window number | the window handle (komorebi's `hwnd`) | +| `rect` | `kCGWindowBounds`, in display points | the visible frame (`DWMWA_EXTENDED_FRAME_BOUNDS`), in pixels | +| `visible` | `kCGWindowIsOnscreen` (false for windows on other spaces) | `IsWindowVisible`; invisible windows and tool windows are not listed | +| `minimized` | always false: the window server does not say | `IsIconic` | +| `cloaked` | absent | `DWMWA_CLOAKED` | +| `border` | set on the layer-0 windows owned by the `komorebi` process, which are komorebi's borders (`border_of` is never set; pair one with its window by rect, as `lag` does) | `komoborder-` tool windows are listed (other tool windows are not), with `border` true, and `border_of` when the id is a window handle (a floating window); a tiled window's border is named for its container, see `windowtrack.rs` | +| `title` | `kCGWindowName` | `GetWindowTextW` | +| status | written against the `objc2` sources and not built or run | built and run | + +### The program's own windows + +The program has one window, with the recorder as its first tab and the recordings as the others. It must not be in the +video it is recording, in either of its sizes (the small always-on-top one and the full one), and komorebi must not tile +it. Every `komorebi-recorder` launch (`gui`, `view`, no command, and `komorebi-recorder-gui.exe`) shares one local socket, +so there is never a second window to exclude. + +| | macOS | Windows | +|---|---|---| +| left out of the video | by name: every process called `komorebi-recorder` is excluded from the capture | each window asks to be left out of captures with `WDA_EXCLUDEFROMCAPTURE` (Windows 10 version 2004 and later) | +| left alone by komorebi | the windows have no title, which komorebi does not manage; there is no ignore rule to rely on, so they are never titled | the windows are titled `komorebi-recorder` only once the recorder has run `komorebic ignore-rule exe` for its executables (or no komorebi runs), and are otherwise untitled; `doctor` says which. The rule is komorebi's live one and is not saved: to keep it, see [recording a bug repro](usage/recording-a-bug-repro.md). Also `KOMOREBI_RECORDER_PROBE_WINDOW=1` makes a window ask komorebi whether it would manage it and write the answer to `komorebi-recorder-window-probe.txt` in the temp directory | +| icon | a slice of watermelon, which the Dock shows while a window is open (eframe sets it as the application's icon image). The program is a bare executable and not an app bundle, so Finder gives the file the generic executable icon | the same picture as the window and taskbar icon while a window is open, and compiled into both executables as a resource, which is what Explorer shows for the files; the installer shows it beside the program in Apps & features | +| executables | `komorebi-recorder` | `komorebi-recorder.exe` (console: run commands with this one) and `komorebi-recorder-gui.exe` (no console window) | + +## Linux (KDE Plasma) + +The Linux host records komorebi for KDE (`komorebi-for-kde`), on Plasma 6.8 or later, on Wayland. It is the +only Linux desktop the recorder knows: the screen comes from KWin's own screencast, and the windows from komorebi for +KDE. What is written from the sources and has not been seen to work on a real Plasma session is listed, one check each, in +[kde-checkpoints.md](dev/kde-checkpoints.md). + +| | Linux (KDE Plasma) | +|---|---| +| master clock | `CLOCK_MONOTONIC`, `host.clock` is `monotonic`; it does not count the time the machine is suspended, and `CLOCK_BOOTTIME`, which does, is the reference clock | +| sleep, wake and lock | heard on D-Bus: logind's `PrepareForSleep` on the system bus and the screen saver's `ActiveChanged` on the session bus; whether the screen is locked is `org.freedesktop.ScreenSaver.GetActive`. The lid and the displays' power are not read; the displays are polled | +| the displays | the compositor's outputs, over Wayland (`wl_output` and `xdg_output`), kept up to date by a thread that reads the compositor's events. A display's `id` is the FNV-1a hash of the output's name, which is komorebi for KDE's id for a monitor with no serial number | +| capturing the screen | KDE's `zkde_screencast_unstable_v1`, which gives a PipeWire node and asks the user nothing; `pipewiresrc` reads it, and the frames are placed in the file's time as on Windows | +| permission | KWin offers the screencast only to a program whose desktop file lists `zkde_screencast_unstable_v1` in `X-KDE-Wayland-Interfaces`. `doctor` prints the file to write, with the path of the executable in it | +| the video | H.264 only, written by GStreamer (`x264enc` and `mp4mux`) as fragmented MP4 in segments, as on the other hosts. HEVC is refused. A display of an odd size, or one H.264 cannot take, is scaled by the pipeline | +| decoding | GStreamer: `decodebin` to BGRA at the size asked for, seeking to the key frame before the time and decoding forward. A frame's time is its stream time, because an MP4's edit list puts the media time of the first frame after zero | +| the OS trace | the journal: `oslog/system.ndjson` (`journalctl --user`, of `kwin_wayland`, `plasmashell` and `komorebi`, stream `journal`) and `oslog/eventlog.ndjson` (every warning and worse, stream `journal-warnings`). An entry's time is `__MONOTONIC_TIMESTAMP`, which is the master clock | +| the window track | komorebi for KDE's own *beliefs*, not KWin's geometry: only windows komorebi manages are listed, with komorebi's rect, each shown when its workspace is showing and it is in front of its stack. **Not done:** listing KWin's real windows. KWin has no D-Bus call that lists windows: `getWindowInfo` needs a UUID, and the only listing the recorder uses is `/WindowsRunner` `Match(query)`, which is a KRunner text search (it may need a minimum query length; not checked, so not relied on) and costs one `getWindowInfo` per window per tick. A complete, cheap source needs a KWin script (`loadScript` over `org.kde.kwin.Scripting`) that pushes `frameGeometryChanged`/`windowAdded` to a D-Bus service of the recorder's, or the `org.komorebi.Bridge` events; that was not built or tried on the Plasma guest. Until then the track on KDE cannot show a window komorebi does not manage, nor a frame that disagrees with komorebi's `rect` (use `--probe-windows` for per-event real frames). komorebi for KDE has no border windows (KWin's rounded-corners effect paints them), so there is nothing to mark `border` there | +| komorebi's sockets | `$XDG_RUNTIME_DIR/komorebi/` (the command socket `komorebi.sock` and the subscribers'), and the data directory where there is no runtime directory | +| komorebi's log | `~/.local/share/komorebi/komorebi_plaintext.log.`; the span of an event's handler is `handle_event` and of a command's `process_command` | +| the program's own windows | left out of the video by a KWin window rule (`excludefromcapture`, forced for the window class `komorebi-recorder`), which `record` writes to `~/.config/kwinrulesrc` and has KWin read; komorebi is told to ignore them by their application id, `komorebic ignore-rule class komorebi-recorder`, run before the window exists. Whether KWin honours the rule for a screencast of an output is a check still to make (K3) | +| the window's application id | the window's application id is `komorebi-recorder`, which is the `class` komorebi for KDE sees | +| `--probe-windows` | on by default, as on Windows: after each window event KWin is asked about the window (`org.kde.KWin.getWindowInfo`: the desktops it is on, minimized, its frame). komorebi's window is found by its class, title and frame through KWin's windows runner, because komorebi's ids are a count and not KWin's UUIDs. A window on a desktop that is not showing is `not visible`, one alone on `komorebi-park` is `cloaked` | +| ghost tiles | found from the frames komorebi has in its state, as on a Mac (a window in front on a showing workspace that has no frame anyone can see). No rule is proposed for one: komorebi for KDE's rules are in its own repository | + +komorebi for KDE's rules are checked against the log and the events of a running komorebi (`komorebi --fake-backend`, an +in-memory desktop with no windows), and against its source. They are not checked against a recording of a real +desktop, so an analysis of a KDE session says so, as it does for Windows. + +### What is needed on the machine + +- the GStreamer plugins that read PipeWire (`pipewire`), decode and encode H.264 (`gst-libav`, `gst-plugins-ugly` for + `x264enc`), and mux MP4 (`gst-plugins-good`); the Nix package puts them on the plugin path; +- `libwayland-client`, which is opened when it is needed, and what the window draws through (`libxkbcommon`, `libGL` or + Vulkan): the Nix package and the dev shell put them on `LD_LIBRARY_PATH`; +- `ffmpeg` for `frames` and `selftest`, and nothing else. + +## Degraded modes + +- `record --no-video` records everything but the screen, on the same clock. It asks the system nothing about displays and + needs no permission. +- `record --no-komorebi` skips touching komorebi altogether, and `--no-oslog` skips the OS trace. +- A komorebi that is not running when a recording starts is not an error: it is noted, and the recorder subscribes when + it appears. +- The `unsupported` host investigates sessions and refuses to record. + +## Advice by audience + +- **Recording on macOS.** Grant Screen Recording to the terminal, and start komorebi with `--log-level trace`. +- **Recording on Linux (KDE Plasma).** Install the desktop file `doctor` prints, so that KWin offers the screencast, and start + komorebi for KDE with `--log-level debug`. Use `record --no-video` where there is no screencast to be had. +- **Recording on Windows.** There is nothing to grant. Use `komorebi-recorder.exe` for commands, since the GUI build + prints nowhere, and start komorebi with `--log-level debug` and `RUST_LOG=komorebi=debug`. +- **Investigating on any host.** A session is read the same everywhere. `ffmpeg` is needed only for `frames` and + `selftest`. +- **Changing code that has a host-specific part.** Provide the function on every host, and run `just check-windows` from a + Mac: it type-checks every `cfg(windows)` line, tests included. diff --git a/docs/recorder/dev/etw-spike.md b/docs/recorder/dev/etw-spike.md new file mode 100644 index 00000000..086d9e7e --- /dev/null +++ b/docs/recorder/dev/etw-spike.md @@ -0,0 +1,91 @@ +# ETW spike: what an ETW tap would tell us about window management + +Date 2026-10-28, Windows 11 26200, **not elevated** shell (user is in Performance Log Users). No recorder +code was written. Traces and scripts live in the session scratchpad, not the repo. + +## Method + +`logman create trace -p 0xffffffffffffffff 0xff -o .etl -ets` for Win32k, User32, Dwm-Core +and Shell-Core (one file session each, plus a real-time `-rt` Win32k session). Then a throwaway +WinForms window owned by a spawned `pwsh` (hwnd 0x2A80880, pid 10404) was driven with `SetWindowPos` +(move, resize, TOPMOST, NOTOPMOST), `ShowWindow` (minimize, restore, hide, show), +`DwmSetWindowAttribute(DWMWA_CLOAK)` and `SetForegroundWindow` over about 6 s. Sessions stopped, decoded +with `tracerpt -of CSV`. Nothing outside that window was touched. + +## Elevation + +- **Every session started without elevation**: Win32k, User32, Dwm-Core, Shell-Core, real-time and file. + Nothing was refused; membership of Performance Log Users is enough for user-mode provider sessions. + (`logman query -ets` listed all five as Running.) We did not try the kernel logger (NT Kernel Logger, + `SystemTraceProvider`); that one does need admin and is out of scope. +- `Microsoft-Windows-User32` {B0AA8734...} started fine but emitted **nothing** (2 header events only; it + has no registered manifest on this machine: `wevtutil gp` cannot find it). It is not a window-API trace. +- If a future machine does refuse, the command to try elevated is the same one: + `logman create trace x -p "{8C416C79-D49B-4F01-A467-E56D3AA8234C}" 0xffffffffffffffff 0xff -o x.etl -ets`. + +## What the events contain + +| Provider | Events in ~22 s | Notes | +|---|---|---| +| Win32k | 102,911 delivered, **817,109 lost** (default buffers) | about 4.7k/s kept, about 42k/s offered, while the desktop was idle | +| Dwm-Core | 216,834, 0 lost | about 10k/s | +| Shell-Core | 48,123, 0 lost | about 2k/s | +| User32 | 0 | nothing | + +- **Win32k** is a graphics and input trace (CreateHandle/DestroyHandle/TransformHandle, FlipManager, + touch targeting, pseudo-device, "Info" ids 164/167/301/65...). 79 events carried our pid in the header + during the run, spread over ids 164, 166, 167, 45, 84-87, 452, 453, 2000, and none of them was a + window-management call. The hwnd 0x2A80880 appears in **no** Win32k payload; neither did our rect + (500x400, 600x450) or SWP flags. Payloads are untyped numbers to `tracerpt` (the manifest is in + win32kbase.sys, see below), so even a mapping would be reverse engineering. +- **Dwm-Core** carried our hwnd in 36 events, only in two kinds, both from `dwm.exe`: + - id 84, task `WINDOWNODE_GDISPRITE_ASSOCIATION`: hwnd, then handles. 26 hits. + - id 209 `Stop` (task 137): hwnd, 0, 0, then a 4x4 identity transform matrix. 10 hits. No rect, no + flags, no cloak bit, no z-order, no show/hide. The 10 hits for roughly 11 API calls cannot be tied to + a call without the timestamp alignment. +- **Shell-Core**: no hwnd or pid match at all for our window. +- **Who called?** Header PID/TID of every Dwm-Core event is `dwm.exe` (0x834) and the thread inside it + (0xA50/0xBFC). That is the *consumer* of the change, never the process that called `SetWindowPos`. For + Win32k the header pid is the process that ran the kernel path (ours, 10404, for our own work), which is + the closest to a caller found, but it showed only graphics events, not "this process moved window X". + +Sample (Dwm id 209, trimmed): +`Microsoft-Windows-Dwm-Core, Stop, 209, ..., pid 0x834, tid 0xBFC, ..., 0x2A80880, 0x0, 0, 1.0 0 0 0 | 0 1.0 0 0 | 0 0 1.0 0 | 0 0 0 1.0` + +## What WinEvents already give + +The existing hook (`EVENT_MIN..EVENT_MAX`, `SetWinEventHook`) yields, per top-level window: hwnd plus +class and program (looked up), `EVENT_OBJECT_LOCATIONCHANGE`, `OBJECT_CLOAKED`/`UNCLOAKED`, +`OBJECT_SHOW`/`HIDE`, `SYSTEM_FOREGROUND`, `SYSTEM_MINIMIZESTART/END`, move/size start/end, +`OBJECT_REORDER`, with the event thread/time. Compared with ETW: WinEvents are *per window* and +semantic, ETW is *per frame* and about DWM internals. ETW added no hwnd-level fact the hook lacks, and +neither source yields the rect or flags (the hook consumer can call `GetWindowRect` at record time, which +is already the cheaper way). + +## Cost of a tap + +- Behind `platform::trace`, as a third tap beside the hook and Event Log. Crate options: `ferrisetw` + (real-time `UserTrace` and a callback, schema decode through TDH) or the `windows` crate + `StartTraceW/EnableTraceEx2/OpenTraceW/ProcessTrace`. About 1.5 to 2 days including dropping + events on a hot path and a session that must be stopped at crash (a leaked session survives the + process; use a unique name and `ControlTrace(STOP)` on startup). +- It fits the oslog record shape (`ts, source, process, message`), but only by flattening each event to + `provider:id task` plus untyped numbers, which is not useful to read. +- Noise is the problem: 10k to 40k events/s from Win32k and Dwm-Core with nothing happening, almost all + of it frame pacing. It would need a per-id allowlist and still shows nothing we want. + +## Recommendation: **don't build it** + +None of the four goals (hwnd, rect, flags, caller) come out of the providers that are available without +elevation. What would give them is not an ETW provider but a komorebi-side change: + +1. Have komorebi log, at the place it applies a layout (its `WindowManager::update_*` and `Window::set_position`, + `hide`, `cloak` paths), the hwnd, target rect, SWP flags, and `reason` (which event or command caused + it) into its own state/notification stream; the recorder already ingests that stream. +2. If the *foreign* caller is wanted (an app fighting the tiling), the practical method is a `WH_CBT` + or `WH_CALLWNDPROC` hook with `WM_WINDOWPOSCHANGING`, which carries the `WINDOWPOS` struct (hwnd, + rect, flags) and runs in the caller's context. That needs a DLL injected and is a much bigger step + than a trace tap; the cheaper step is to take a `GetWindowRect` snapshot on each ObjectLocationChange + in the recorder (a few extra lines in `win/trace.rs`) and record the flags komorebi sets itself. +3. Revisit ETW only if a Microsoft-Windows-Win32k manifest with window-position events turns up in a + later Windows build, or if we accept the kernel logger (admin) for a one-off diagnostic. \ No newline at end of file diff --git a/docs/recorder/dev/kde-checkpoints.md b/docs/recorder/dev/kde-checkpoints.md new file mode 100644 index 00000000..e34a2e0d --- /dev/null +++ b/docs/recorder/dev/kde-checkpoints.md @@ -0,0 +1,117 @@ +# KDE checkpoints + +`komorebi-recorder` is built and tested on Linux, but with no KWin to talk to: what touches a real Plasma session was +written from KWin's and komorebi for KDE's sources and has not been seen running. Each checkpoint below is a short list +of commands to run on a Plasma 6.8 session that runs komorebi for KDE, what to expect, and what to send back. The point +of each is to replace something read in a source with something observed. + +Send back, for any step that does not behave: the exact command, what it printed, the output of +`komorebi-recorder doctor --json`, and the session directory if there is one. + +## What has been seen + +Against komorebi for KDE's in-memory desktop (`komorebi --fake-backend`, which has no windows) and the Wayland compositor +of WSLg (which has outputs and no screencast): + +- `doctor` names komorebi for KDE and finds its socket in `$XDG_RUNTIME_DIR/komorebi`; +- `record --no-video` records its log, its commands as events with the handler of each, and the journal in two streams, + and `events`, `summary` and `findings` read the session with the KDE rules; +- the displays are listed from the compositor, with a size and a position; +- a made-up source of frames is written to an H.264 file, which the decoder reads back, at the size of the display and + at an odd size. + +And in the Plasma guest of komorebi for KDE (`nix run .#run-kde`, KWin 6.8.2, komorebi with windows, headless over ssh; +the recorder from `nix build` runs in it, as the guest mounts the host's store): + +- **K1:** `doctor` says `permitted: yes` and a recording works with no desktop file written. Whether KWin offers the + screencast to every program, or this guest's configuration allows it, was not found out. +- **K2:** a recording of the output is a playable H.264 file with the windows, the panel and the cursor in it, and its + length is the recording's. The frames' times were wrong at first (the GStreamer pipeline took the clock of the + PipeWire stream, which starts at zero) and are on the master clock now. The log, the events and their handlers line up. +- **K3:** the rule that leaves the recorder's window out of the capture works: a window of the class + `komorebi-recorder` that had the focus and was on top is not in the video, and one of another class is. KWin 6.8 + rewrites `kwinrulesrc` with `Order=` in place of `count=` and `rules=`, and the recorder extends whichever the file has. +- **`--probe-windows`:** a window of an event is found in KWin and its facts are read: frame, minimized, the desktops. The + probes of a recording with workspace switches, a new window, a stack and a closed window agree with komorebi, and the + one of a window that was closed says it no longer exists. komorebi's workspaces are KWin's virtual desktops, so a + window of another workspace is not on the desktop that shows; the park desktop holds only windows hidden behind a + front one. What was not seen: a probe that disagrees with komorebi (nothing was broken), and a window that cannot be + told from another by class, title and frame (it has no probe). + +## Getting the binary + +```sh +nix build .#komorebi-recorder --out-link result +result/bin/komorebi-recorder doctor +``` + +The package wraps the program so that GStreamer finds its plugins (`gst-libav`, `x264enc`, the PipeWire source) and the +program finds `libwayland-client`. Outside Nix, install the same. + +## K0: what komorebi for KDE really sends + +Status: **can be run now, and is the most useful one.** The rules that read komorebi for KDE's events, `State` and log +were written from its source and checked against its log and commands, not against windows (the analysis says so in a +warning until they have been). + +1. Start komorebi for KDE with debug logging: `komorebi --log-level debug` (or `RUST_LOG=komorebi=debug`). +2. `komorebi-recorder doctor`. Expect `komorebi for KDE`, `running: yes`, the socket under `$XDG_RUNTIME_DIR/komorebi` + answering a state query, the log directory `~/.local/share/komorebi` with a `komorebi_plaintext.log.` in it, your + outputs, and `clock: Monotonic`. +3. Record 60 seconds with `--no-video`, and during them: open and close a few windows of different programs, focus + between them, drag one to a new place and drag-resize one, stack and unstack, switch workspaces + (`komorebic focus-workspace 1` and back), and lock and unlock the screen. +4. `komorebi-recorder events `. Expect each window event with its window id, program and notification + (`WindowAdded`, `InteractiveMoveFinished`, ...), each with a handler. Send back any event named `unreadable` or + classified `other`, any window event with no handler, and any `MoveEnd` that is not classified `drag`. + +## K1: the screencast is offered, and what it asks + +Status: not run. This is the one the Linux host depends on. + +1. `komorebi-recorder doctor`: under `capture`, `permitted: no` and the note with the desktop file. +2. Write `~/.local/share/applications/komorebi-recorder.desktop` as the note says (`Exec=` is the path of the executable + that runs), and start the recorder again. Expect `permitted: yes`. In the Nix package the program + is a wrapper that starts `.komorebi-recorder-wrapped`, and that is the executable the compositor sees, so it is the path + `doctor` prints (it is `current_exe`). +3. If it is still `no`: say where KWin looks for desktop files and how it matches the program to one. Whether a + desktop file in the user's `applications` directory is enough, whether `Exec=` has to be the executable's real path, + and whether a program started from a terminal is matched at all, are what this finds out. If it cannot be made to + work, the screencast portal (`xdg-desktop-portal`) is the way around it, behind the same `CaptureBackend`. + +## K2: recording the screen + +Status: not run. + +1. `komorebi-recorder displays`. Expect each output with its size and position in the layout (logical pixels). Compare the + ids with komorebi's: `komorebic monitor-information` (they agree for a monitor with no serial number). +2. `komorebi-recorder record --duration 20`, and move windows during it. Expect a `video-.mp4` for each output and + `frames-.csv`, and in the session summary no error. +3. Play it with `komorebi-recorder view `. Expect the picture to move with the windows, and the cursor in it. +4. `komorebi-recorder doctor --json`: the clock check (`latency`) of the segments should be near zero. A large number + means the frames' times are not on the master clock, which would be the PipeWire timestamps: send the numbers back. +5. Leave the screen still for ten seconds: the video should still advance (the last frame is repeated). +6. Unplug a monitor, or turn one off, if there is one to spare: the segment ends and a new one starts when it returns. +7. Suspend and resume: every output starts a new segment. + +## K3: the program's own window + +Status: not run. + +1. `komorebi-recorder` with no command opens the window. Expect it to be titled `komorebi-recorder` and not tiled. +2. If it was tiled, `komorebic ignore-rule class komorebi-recorder` was not applied in time, or the window's class is not + what is expected: `komorebic state` says what the window's class is. +3. `record` writes a KWin window rule (`~/.config/kwinrulesrc`, group `komorebi-recorder-exclude-from-capture`) that + forces `excludefromcapture` for the class `komorebi-recorder`. Record with the window open and play the video: the + window should not be in it. If it is, say whether `kwinrulesrc` has the rule, whether System Settings > Window + Rules shows it, and what `qdbus6 org.kde.KWin /KWin supportInformation` says of the window's class. + +## K4: the OS trace and the window track + +Status: the journal was followed on a machine with no Plasma. + +1. `record --duration 30`, and cause some noise: open a program that is not in the session, plug something in. + Expect `oslog/system.ndjson` to have `kwin_wayland` and `plasmashell` entries, and `oslog/eventlog.ndjson` the warnings. + A user who is not in the `systemd-journal` group may see none of the system's: the problems say so. +2. `record --duration 30` (the window track is on by default). Expect `windows.jsonl` with the windows komorebi manages, a window behind a + stack not shown, and a window of another workspace not shown. diff --git a/docs/recorder/dev/windows-checkpoints.md b/docs/recorder/dev/windows-checkpoints.md new file mode 100644 index 00000000..f80cead7 --- /dev/null +++ b/docs/recorder/dev/windows-checkpoints.md @@ -0,0 +1,216 @@ +# Windows checkpoints + +`komorebi-recorder` is built for Windows from a Mac (and, in CI, on Windows), so nothing that touches a real +Windows desktop has been seen running. Each checkpoint below is a short list of commands to run on a Windows PC or +VM, what to expect, and what to send back. The point of each is to replace something read in +komorebi's source with something observed. + +Send back, for any step that does not behave: the exact command, what it printed, the output of +`komorebi-recorder.exe doctor --json`, and the session directory if there is one. + +## Getting the binaries + +On the Mac, in this repository: + +```sh +nix build .#komorebi-recorder-windows --out-link result-windows +ls result-windows/bin +# komorebi-recorder.exe console build: use this one to run commands and read their output +# komorebi-recorder-gui.exe the same program without a console window (shortcuts, the viewer) +``` + +Copy `komorebi-recorder.exe` to the Windows machine. The `windows` job of CI builds the same executables natively and +uploads them, with the installer, as artifacts of the run. To check that the Windows code still type-checks +without building it: `nix flake check` (the `komorebi-recorder-windows-clippy` check) or +`nix develop .#windows --command cargo clippy --all-targets`. + +## W0: what komorebi for Windows really sends + +Status: **can be run now, and is the most useful one.** Reading komorebi for Windows' events, state +and logs was written from komorebi's source and has not been checked against a recording of it (the +analysis says so in a warning until it has); this is what checks it. + +1. Start komorebi for Windows with debug logging. The default of `RUST_LOG=debug` also prints every + dependency, so name the crate: + + ```powershell + komorebi.exe --log-level debug + ``` + + (or set `RUST_LOG=komorebi=debug` first). + +2. Describe the machine: + + ```powershell + komorebi-recorder.exe doctor + komorebi-recorder.exe doctor --json > doctor.json + ``` + + Expect: `komorebi for Windows`, `running: yes`, a binary path and version, `log level: debug`, + `answers a state query: yes`, the log directory under `%TEMP%` with a `komorebi_plaintext.log.` + in it, your displays with their scale, and `clock: Qpc`. `can record: yes` and `can decode: yes` + are what this build claims; whether they are true is W1 and W3. Anything under "things to know" + is worth reading. If the state query is + reported as answered with nothing, say what komorebi was doing (paused, another virtual desktop). + +3. Record 60 seconds without video, and during them: + + - focus a few windows of different programs (click, and with `komorebic.exe focus left|right`); + - drag a window to a new place, and drag-resize one; + - move windows with komorebic (`move`, `stack`, `unstack`, `focus-workspace 1` and back); + - open and close a program, and minimize and restore one; + - if you can: change the display scaling or resolution, connect or disconnect a monitor, switch to + another virtual desktop and back, lock (Win+L) and unlock once. + + ```powershell + komorebi-recorder.exe record --no-video --duration 60 --out C:\temp\w0 + ``` + + Expect a line per second with `events` counting up as you act, `komorebi log` growing and `OS + trace` counting up (WinEvents for every window, so it is busy while you work), then `session: + C:\temp\w0`. If the OS trace is too much or too little, say so: it is folded per window but its + volume on a real machine has not been seen. `--no-oslog` leaves it out. + +4. Look at it, to see what the current (macOS-vocabulary) analysis makes of it. It is expected to be + wrong in places, and what is wrong is the point: + + ```powershell + komorebi-recorder.exe brief C:\temp\w0 + komorebi-recorder.exe events C:\temp\w0 --json > events.json + ``` + +5. Send back `doctor.json` and the whole `C:\temp\w0` folder, zipped. It contains the titles and + program names of your windows (`events.jsonl`, `states\*.json`, `komorebi.log`); redact them first + if that matters (they only need to still be valid JSON, and the Windows fixtures are scrubbed by + hand anyway). + +Also worth one run, for the checks komorebi's state cannot make (it hides windows by cloaking them +in place): record with the probe on, and switch workspaces a few times (`komorebic.exe +focus-workspace 1`, `2`, `1`) while focusing windows. + +```powershell +komorebi-recorder.exe record --probe-windows --no-video --duration 60 --out C:\temp\w0-probe +komorebi-recorder.exe overview C:\temp\w0-probe +``` + +`overview` should report nothing about hidden windows if komorebi and Windows agree. If it reports +`visible-window-hidden` or `hidden-window-visible`, `komorebi-recorder.exe trace C:\temp\w0-probe +event:` shows what Windows said; say whether it is a real fault or a false alarm (a window mid +transition, say), and how many there were. The probe asks komorebi about a window by `DebugWindow`; +if komorebi stalls while it runs, say so. + +Optional, for the part of the plan that stops the recorder window being tiled: for one window komorebi +manages and one it does not (get handles from `komorebic.exe state`), run + +```powershell +komorebi-recorder.exe doctor --probe window --hwnd +``` + +and send what komorebi says. + +## W1: capture and the clock across a sleep + +Status: **can be run now.** Windows capture, the power watcher and the OS trace are built but have +never run on Windows; these are the first runs. + +1. The clock across a sleep (see above): `doctor --probe clock --seconds 120`, sleeping the machine + for about a minute while it runs. + +2. Capture on a static screen. Leave the screen completely still for most of it: + + ```powershell + komorebi-recorder.exe doctor --probe capture --seconds 20 + ``` + + Expect, per display, a segment with a handful of frames, `idle callbacks 0` (Windows says nothing + about a still screen), a file that `plays` for about the length recorded, and `latency` of a few + milliseconds to well under a second, `(NOT on the host clock)` absent. Check it with ffmpeg if you + have it: `ffprobe \video-.mp4`. + +3. Two monitors, if you have them: the same again, and say what it printed for each. + +4. Kill it and play the file. Start a longer one, then end it without letting it stop: + + ```powershell + komorebi-recorder.exe doctor --probe capture --seconds 60 + # after 15 s, from another window: + taskkill /F /IM komorebi-recorder.exe + ``` + + Then try to play `video-*.mp4` in the session folder (any player), and run + `komorebi-recorder.exe reindex `. This answers whether Media Foundation's fragmented MP4 + survives a kill (the part before the last fragment, about a second, should play) and whether + `reindex` can recover the session. Say what the player did and what `reindex` printed. + +## W2: a real recording, and the machine doing the things + +Status: can be run once W1 shows capture works. + +```powershell +komorebi-recorder.exe selftest --seconds 8 +komorebi-recorder.exe record --duration 60 --out C:\temp\w2 +komorebi-recorder.exe reindex C:\temp\w2 +``` + +`selftest` needs `ffmpeg` on PATH; expect `PASS` with the worst disagreement under 100 ms. During the +60 s recording: dock or undock a display, put the machine to sleep and wake it, lock and unlock +(Win+L), and close the lid if it is a laptop. Afterwards `komorebi-recorder.exe summary C:\temp\w2` +and `komorebi-recorder.exe at C:\temp\w2