mirror of
https://github.com/ryan4yin/nix-config.git
synced 2026-11-13 12:13:36 +01:00
feat(computer-use): sync the X clipboard and document the input-level design (#308)
- run two autocutsel instances so the X PRIMARY/CLIPBOARD selections stay in sync, making copy/paste work in the session and through VNC - drop --disable-blink-features=AutomationControlled: nothing drives the browser over CDP, so navigator.webdriver is already false and the flag only marked the setup as automated - document why the driver stays on real X11/AT-SPI input and is never granted a CDP attach, and note the SSH-tunnel path for manual VNC access - extend the computer-use eval test
This commit is contained in:
1 parent
b19818ef44
commit
3b13291216
4 files changed
+106
-6
No files matched your search
@@ -14,17 +14,40 @@ The session is built on Xvfb plus i3 because the drivers support native X11 at t
|
||||
(cua-driver "Linux X11 — Supported"; computer-use-linux has an i3 backend). cua-driver's Sway lane
|
||||
is XWayland/X11-based, so a native X11 session is simpler and better supported than Sway/Wayland.
|
||||
|
||||
## Input-level control, not CDP
|
||||
|
||||
The point of this environment is to look like a real user device, so the agent drives the browser
|
||||
the same way a person would: real X11/AT-SPI input (`click`, `type_text`, `press_key`, screenshot,
|
||||
accessibility tree). It does **not** use cua-driver's CDP-backed `browser_*` tools, because
|
||||
attaching to the existing profile:
|
||||
|
||||
- needs a launch grant (`--grant existing-profile`, or unrestricted mode), and
|
||||
- turns on a **persistent, page-detectable** remote-debugging endpoint, and
|
||||
- can read data that never renders (cookies, storage) rather than only what is on screen.
|
||||
|
||||
So `cua-driver serve` is started with **no** launch grants here, and no `--remote-debugging-port` is
|
||||
passed to the browsers (see `browserFlags`). Keep it that way: adding either re-introduces the CDP
|
||||
surface this design avoids.
|
||||
|
||||
No automation switches are passed (`--enable-automation`,
|
||||
`--disable-blink-features=AutomationControlled`, `--remote-debugging-port`): nothing here drives the
|
||||
browser over CDP, so `navigator.webdriver` is already false. The one flag with a realism cost is
|
||||
`--force-renderer-accessibility` — few real users force a11y on — but it is required for the AT-SPI
|
||||
tree the agent reads, so it stays.
|
||||
|
||||
## How it is wired
|
||||
|
||||
- `home/linux/gui/i3/default.nix` defines the option `modules.desktop.computerUse` and the user
|
||||
services: `xvfb` → `i3` → (`x11vnc`, `cua-driver serve`).
|
||||
services: `xvfb` → `i3` → (`x11vnc`, `autocutsel-primary` / `autocutsel-clipboard`,
|
||||
`cua-driver serve`).
|
||||
- `modules/nixos/desktop/computer-use.nix` provides the system side (`linger`, AT-SPI, fonts, Mesa,
|
||||
Clash Verge proxy, proxy-region timezone).
|
||||
- the generated `computer-use-init` script (exec'd by i3) publishes `DISPLAY`/`XDG_SESSION_TYPE` to
|
||||
the systemd user manager, clears any stale `WAYLAND_DISPLAY`, and enables the AT-SPI bridge via a
|
||||
gsettings key.
|
||||
- Drivers live in `overlays/cua-driver.nix` and `overlays/computer-use-linux.nix`.
|
||||
- Host wiring: `home/hosts/linux/idols-ruby.nix`, `home/hosts/linux/idols-kana.nix`.
|
||||
- Host wiring: `home/hosts/linux/idols-ruby.nix`, `home/hosts/linux/idols-kana.nix`,
|
||||
`hosts/idols-ruby/computer-use.nix`.
|
||||
|
||||
## The i3 config
|
||||
|
||||
@@ -36,5 +59,22 @@ The generated config is intentionally minimal:
|
||||
- `exec --no-startup-id .../computer-use-init` — one-time session setup, see above.
|
||||
|
||||
There are no keybindings, bars or autostarted apps: agents drive the session through the drivers,
|
||||
not the keyboard. Manual inspection is done over VNC (`x11vnc`, bound to `127.0.0.1`; reach it
|
||||
through an SSH tunnel).
|
||||
not the keyboard.
|
||||
|
||||
## Manual access over VNC
|
||||
|
||||
`x11vnc` binds `127.0.0.1:5900` with no password, so manual inspection goes through an SSH tunnel
|
||||
rather than exposing the port. Remmina's VNC profile has an **SSH Tunnel** tab for this; the manual
|
||||
equivalent is:
|
||||
|
||||
```bash
|
||||
ssh -L 5900:127.0.0.1:5900 <host>
|
||||
# then point the VNC client at 127.0.0.1:5900
|
||||
```
|
||||
|
||||
## Clipboard
|
||||
|
||||
Xvfb has no clipboard of its own and an X11 selection lives only as long as the application that
|
||||
owns it, so without help a VNC client's clipboard sync is unreliable. Two `autocutsel` instances
|
||||
mirror the `PRIMARY` and `CLIPBOARD` selections, which makes copy/paste work both inside the session
|
||||
and between it and the VNC client.
|
||||
@@ -15,9 +15,18 @@ let
|
||||
# harden the browser. The browser goes out through Clash Verge's proxy, so
|
||||
# non-proxied WebRTC UDP must stay disabled (a real IP leak would contradict
|
||||
# the exit region); everything else is a stock-looking setting.
|
||||
#
|
||||
# Deliberately absent: every automation/CDP switch. No
|
||||
# `--remote-debugging-port` (a persistent, page-detectable endpoint) and no
|
||||
# `--enable-automation` / `--disable-blink-features=AutomationControlled`.
|
||||
# Nothing here drives the browser over CDP, so `navigator.webdriver` is
|
||||
# already false without a flag that only marks the setup as automated.
|
||||
browserFlags = [
|
||||
"--force-webrtc-ip-handling-policy=disable_non_proxied_udp"
|
||||
"--disable-blink-features=AutomationControlled"
|
||||
# Required for the AT-SPI accessibility tree the driver reads; the driver
|
||||
# only walks the tree, it cannot make Chromium build one. This is the one
|
||||
# flag with a realism cost (few real users force a11y on) and stays only
|
||||
# because dropping it risks an empty tree.
|
||||
"--force-renderer-accessibility"
|
||||
"--lang=en-US"
|
||||
"--window-size=1920,1080"
|
||||
@@ -70,6 +79,7 @@ in
|
||||
kitty # terminal (supports X11 and Wayland)
|
||||
xdotool # XTEST input (fallback / manual use)
|
||||
xclip # clipboard
|
||||
autocutsel # keep the X PRIMARY and CLIPBOARD selections in sync
|
||||
xvfb # headless X server
|
||||
]
|
||||
++ [
|
||||
@@ -114,7 +124,46 @@ in
|
||||
};
|
||||
};
|
||||
|
||||
# Optional VNC server for manual access; localhost only, use an SSH tunnel.
|
||||
# Xvfb has no clipboard of its own, and an X11 selection lives only as long
|
||||
# as the app that owns it. Mirror PRIMARY and CLIPBOARD with two autocutsel
|
||||
# instances so a VNC client's clipboard (which syncs a single selection)
|
||||
# stays usable in both directions.
|
||||
systemd.user.services.autocutsel-primary = {
|
||||
Unit = {
|
||||
Description = "autocutsel (PRIMARY)";
|
||||
After = [ "xvfb.service" ];
|
||||
Wants = [ "xvfb.service" ];
|
||||
};
|
||||
Install.WantedBy = [ "default.target" ];
|
||||
Service = {
|
||||
Type = "simple";
|
||||
Environment = [ "DISPLAY=${display}" ];
|
||||
ExecStart = "${pkgs.autocutsel}/bin/autocutsel -selection PRIMARY";
|
||||
Restart = "always";
|
||||
RestartSec = 2;
|
||||
};
|
||||
};
|
||||
|
||||
systemd.user.services.autocutsel-clipboard = {
|
||||
Unit = {
|
||||
Description = "autocutsel (CLIPBOARD)";
|
||||
After = [ "xvfb.service" ];
|
||||
Wants = [ "xvfb.service" ];
|
||||
};
|
||||
Install.WantedBy = [ "default.target" ];
|
||||
Service = {
|
||||
Type = "simple";
|
||||
Environment = [ "DISPLAY=${display}" ];
|
||||
ExecStart = "${pkgs.autocutsel}/bin/autocutsel -selection CLIPBOARD";
|
||||
Restart = "always";
|
||||
RestartSec = 2;
|
||||
};
|
||||
};
|
||||
|
||||
# Optional VNC server for manual access. Localhost only: connect through an
|
||||
# SSH tunnel (Remmina's VNC profile has an SSH Tunnel tab; plain
|
||||
# `ssh -L 5900:127.0.0.1:5900` works too), so the port is never exposed and
|
||||
# no VNC password is needed.
|
||||
systemd.user.services.x11vnc = lib.mkIf cfg.vnc {
|
||||
Unit = {
|
||||
Description = "x11vnc (localhost) for manual access";
|
||||
@@ -131,6 +180,12 @@ in
|
||||
};
|
||||
|
||||
# cua-driver daemon, so `cua-driver mcp` / `call` can reach it.
|
||||
#
|
||||
# Deliberately a plain `serve` with no launch grants. The driver must drive
|
||||
# the browser with real X11/AT-SPI input, never its CDP-backed `browser_*`
|
||||
# tools: attaching to the existing profile needs `--grant existing-profile`
|
||||
# (or unrestricted mode) and turns on a persistent, detectable
|
||||
# remote-debugging endpoint. Do not add one here. See README.md.
|
||||
systemd.user.services.cua-driver = lib.mkIf cfg.cuaDriver {
|
||||
Unit = {
|
||||
Description = "cua-driver daemon";
|
||||
|
||||
Reference in new issue
Block a user