diff --git a/home/README.md b/home/README.md index a85503c0..fbf26ba4 100644 --- a/home/README.md +++ b/home/README.md @@ -22,6 +22,7 @@ home/ ├── linux/ # Linux-specific home manager configurations │ ├── base/ # Linux base configurations │ ├── gui/ # Linux GUI applications +│ │ ├── i3/ # i3 — headless computer-use session (see i3/README.md) │ │ ├── niri/ # Niri window manager │ │ └── ... │ ├── editors/ # Linux-specific editors @@ -42,6 +43,7 @@ home/ 2. **linux**: Linux-specific configuration - Desktop environments (Noctalia Shell, Niri compositor) + - Headless computer-use session (`gui/i3`, Xvfb + i3; see `gui/i3/README.md`) - Linux-specific GUI applications - System integration tools diff --git a/home/hosts/linux/idols-kana.nix b/home/hosts/linux/idols-kana.nix index a67103bf..a6da9140 100644 --- a/home/hosts/linux/idols-kana.nix +++ b/home/hosts/linux/idols-kana.nix @@ -1,3 +1,11 @@ { - imports = [ ../../linux/core.nix ]; + imports = [ + ../../linux/core.nix + ../../linux/gui/i3 + ]; + + # Headless X11/i3 session for computer use. + modules.desktop.computerUse.enable = true; + # localhost-only VNC server for manual access (use an SSH tunnel). + modules.desktop.computerUse.vnc = true; } diff --git a/home/hosts/linux/idols-ruby.nix b/home/hosts/linux/idols-ruby.nix index a67103bf..4577dfdf 100644 --- a/home/hosts/linux/idols-ruby.nix +++ b/home/hosts/linux/idols-ruby.nix @@ -1,3 +1,13 @@ { - imports = [ ../../linux/core.nix ]; + imports = [ + ../../linux/core.nix + ../../linux/gui/i3 + ]; + + # Headless X11/i3 session for computer use. + modules.desktop.computerUse.enable = true; + # localhost-only VNC server for manual access (use an SSH tunnel). + modules.desktop.computerUse.vnc = true; + # trycua/cua driver, grey-tested on ruby only. + modules.desktop.computerUse.cuaDriver = true; } diff --git a/home/linux/gui/i3/README.md b/home/linux/gui/i3/README.md new file mode 100644 index 00000000..b9d6de2d --- /dev/null +++ b/home/linux/gui/i3/README.md @@ -0,0 +1,39 @@ +# i3 — headless computer-use session + +This module provides the headless X11 session used by the computer-use VMs (`idols-ruby`, +`idols-kana`). It is **not** a normal desktop configuration and is not meant to be used +interactively by a person. + +## Purpose + +Give a computer-use agent a virtual 1920x1080 X display it can observe and control: screenshots, +window discovery/activation, synthetic input, and the AT-SPI accessibility tree. The agent itself +runs elsewhere and connects over MCP; this module is only the "computer" side. + +The session is built on Xvfb plus i3 because the drivers support native X11 at their highest tier +(cua-driver "Linux X11 — Supported"; computer-use-linux has an i3 backend). cua-driver's Sway lane +is XWayland/X11-based, so a native X11 session is simpler and better supported than Sway/Wayland. + +## How it is wired + +- `home/linux/gui/i3/default.nix` defines the option `modules.desktop.computerUse` and the user + services: `xvfb` → `i3` → (`x11vnc`, `cua-driver serve`). +- `modules/nixos/desktop/computer-use.nix` provides the system side (`linger`, AT-SPI, fonts, Mesa, + Clash Verge proxy, proxy-region timezone). +- `computer-use-init.sh` (exec'd by i3) publishes `DISPLAY`/`XDG_SESSION_TYPE` to the systemd user + manager, clears any stale `WAYLAND_DISPLAY`, and enables the AT-SPI bridge via a gsettings key. +- Drivers live in `overlays/cua-driver.nix` and `overlays/computer-use-linux.nix`. +- Host wiring: `home/hosts/linux/idols-ruby.nix`, `home/hosts/linux/idols-kana.nix`. + +## The i3 config + +The generated config is intentionally minimal: + +- `font pango:monospace 10` +- `default_border normal` — keeps title bars, so the client area is slightly smaller than the + screen. That matches a real desktop and avoids a full-screen-sized viewport. +- `exec --no-startup-id .../computer-use-init.sh` — one-time session setup, see above. + +There are no keybindings, bars or autostarted apps: agents drive the session through the drivers, +not the keyboard. Manual inspection is done over VNC (`x11vnc`, bound to `127.0.0.1`; reach it +through an SSH tunnel). diff --git a/home/linux/gui/i3/default.nix b/home/linux/gui/i3/default.nix new file mode 100644 index 00000000..5f453ef7 --- /dev/null +++ b/home/linux/gui/i3/default.nix @@ -0,0 +1,175 @@ +{ + pkgs, + pkgs-x64, + config, + lib, + ... +}: +let + cfg = config.modules.desktop.computerUse; + + display = ":0"; + screen = "1920x1080x24"; + + # Keep this list minimal: the goal is to look like a real user device, not to + # harden the browser. The browser goes out through Clash Verge's proxy, so + # non-proxied WebRTC UDP must stay disabled (a real IP leak would contradict + # the exit region); everything else is a stock-looking setting. + browserFlags = [ + "--force-webrtc-ip-handling-policy=disable_non_proxied_udp" + "--disable-blink-features=AutomationControlled" + "--force-renderer-accessibility" + "--lang=en-US" + "--window-size=1920,1080" + "--window-position=0,0" + "--no-first-run" + "--no-default-browser-check" + ]; + + # Exec'd by i3 once the X session is up. + initScript = pkgs.writeShellScript "computer-use-init" '' + set -eu + + # Make the X session visible to the systemd/dbus user manager, so MCP + # servers and other user services started later inherit DISPLAY. + export DISPLAY=${display} + ${pkgs.dbus}/bin/dbus-update-activation-environment --systemd \ + DISPLAY XDG_SESSION_TYPE XDG_CURRENT_DESKTOP || true + + # This is an X11 session: drop any stale Wayland variables (left over from + # a previous session), otherwise tools such as x11vnc refuse to start. + ${pkgs.systemd}/bin/systemctl --user unset-environment WAYLAND_DISPLAY || true + + # Enable the AT-SPI bridge so computer-use agents can read accessibility + # trees. at-spi-bus-launcher derives org.a11y.Status.IsEnabled from this key. + GSETTINGS_BACKEND=dconf \ + DBUS_SESSION_BUS_ADDRESS="unix:path=''${XDG_RUNTIME_DIR}/bus" \ + ${pkgs.glib.bin}/bin/gsettings set org.gnome.desktop.interface toolkit-accessibility true \ + >/dev/null 2>&1 || true + + ${lib.optionalString cfg.cuaDriver '' + # cua-driver sends content-free telemetry by default; turn it off. + ${pkgs-x64.cua-driver}/bin/cua-driver telemetry disable >/dev/null 2>&1 || true + ''} + ''; +in +{ + options.modules.desktop.computerUse = { + enable = lib.mkEnableOption "headless X11/i3 session for computer use"; + vnc = lib.mkEnableOption "a localhost-only x11vnc server for manual access"; + cuaDriver = lib.mkEnableOption "the trycua/cua driver (cua-driver)"; + }; + + config = lib.mkIf cfg.enable { + home.packages = + with pkgs; + [ + glib # gsettings, to enable the AT-SPI bridge + gsettings-desktop-schemas # the toolkit-accessibility schema + i3 + kitty # terminal (supports X11 and Wayland) + xdotool # XTEST input (fallback / manual use) + xclip # clipboard + xvfb # headless X server + ] + ++ [ + # MCP server / CLI that drives the desktop (see overlays/computer-use-linux.nix). + pkgs-x64.computer-use-linux + ] + ++ lib.optional cfg.vnc pkgs.x11vnc + ++ lib.optional cfg.cuaDriver pkgs-x64.cua-driver; + + # Headless X server. + systemd.user.services.xvfb = { + Unit = { + Description = "Xvfb headless X server"; + }; + Install.WantedBy = [ "default.target" ]; + Service = { + Type = "simple"; + ExecStart = "${pkgs.xvfb}/bin/Xvfb ${display} -screen 0 ${screen} -nolisten tcp"; + Restart = "always"; + RestartSec = 2; + }; + }; + + # Window manager. + systemd.user.services.i3 = { + Unit = { + Description = "i3 window manager (headless X)"; + After = [ "xvfb.service" ]; + Requires = [ "xvfb.service" ]; + }; + Install.WantedBy = [ "default.target" ]; + Service = { + Type = "simple"; + Environment = [ + "DISPLAY=${display}" + "XDG_SESSION_TYPE=x11" + "XDG_CURRENT_DESKTOP=i3" + ]; + ExecStart = "${pkgs.i3}/bin/i3"; + Restart = "always"; + RestartSec = 2; + }; + }; + + # Optional VNC server for manual access; localhost only, use an SSH tunnel. + systemd.user.services.x11vnc = lib.mkIf cfg.vnc { + Unit = { + Description = "x11vnc (localhost) for manual access"; + After = [ "xvfb.service" ]; + Wants = [ "xvfb.service" ]; + }; + Install.WantedBy = [ "default.target" ]; + Service = { + Type = "simple"; + ExecStart = "${pkgs.x11vnc}/bin/x11vnc -display ${display} -localhost -nopw -forever -shared -rfbport 5900"; + Restart = "always"; + RestartSec = 2; + }; + }; + + # cua-driver daemon, so `cua-driver mcp` / `call` can reach it. + systemd.user.services.cua-driver = lib.mkIf cfg.cuaDriver { + Unit = { + Description = "cua-driver daemon"; + After = [ + "xvfb.service" + "i3.service" + ]; + Wants = [ "xvfb.service" ]; + }; + Install.WantedBy = [ "default.target" ]; + Service = { + Type = "simple"; + Environment = [ + "DISPLAY=${display}" + "XDG_SESSION_TYPE=x11" + ]; + ExecStart = "${pkgs-x64.cua-driver}/bin/cua-driver serve"; + Restart = "always"; + RestartSec = 2; + }; + }; + + xdg.configFile."i3/config".text = '' + # i3 configuration for the headless computer-use session. + # Keep it minimal; see README.md in this directory for the rationale. + + font pango:monospace 10 + default_border normal + + exec --no-startup-id ${initScript} + ''; + + programs.brave-origin = { + enable = true; + commandLineArgs = browserFlags; + }; + programs.chromium = { + enable = true; + commandLineArgs = browserFlags; + }; + }; +} diff --git a/hosts/idols-kana/default.nix b/hosts/idols-kana/default.nix index 0df9270c..593de470 100644 --- a/hosts/idols-kana/default.nix +++ b/hosts/idols-kana/default.nix @@ -18,6 +18,7 @@ in { imports = [ ../idols-ruby/packages.nix + ../idols-ruby/computer-use.nix ../idols-ruby/oci-containers ]; diff --git a/hosts/idols-ruby/computer-use.nix b/hosts/idols-ruby/computer-use.nix new file mode 100644 index 00000000..fab73e0f --- /dev/null +++ b/hosts/idols-ruby/computer-use.nix @@ -0,0 +1,4 @@ +{ + # Headless computer-use environment (sway + desktop automation + browser). + modules.desktop.computerUse.enable = true; +} diff --git a/hosts/idols-ruby/default.nix b/hosts/idols-ruby/default.nix index 2ceca538..cded3243 100644 --- a/hosts/idols-ruby/default.nix +++ b/hosts/idols-ruby/default.nix @@ -18,6 +18,7 @@ in { imports = [ ./packages.nix + ./computer-use.nix ./oci-containers ]; diff --git a/modules/nixos/desktop/computer-use.nix b/modules/nixos/desktop/computer-use.nix new file mode 100644 index 00000000..43d22024 --- /dev/null +++ b/modules/nixos/desktop/computer-use.nix @@ -0,0 +1,58 @@ +{ + config, + lib, + pkgs, + pkgs-master, + myvars, + ... +}: +let + cfg = config.modules.desktop.computerUse; + username = myvars.username; + + # Keep the guest timezone consistent with the proxy's exit region. A + # China-based timezone with a US/JP exit IP is a strong automation signal. + timeZone = if cfg.proxyRegion == "jp" then "Asia/Tokyo" else "America/Los_Angeles"; +in +{ + options.modules.desktop.computerUse = { + enable = lib.mkEnableOption "headless computer-use environment (X11 + desktop automation)"; + proxyRegion = lib.mkOption { + type = lib.types.enum [ + "us" + "jp" + ]; + default = "us"; + description = "Proxy exit region; keeps the timezone consistent with the egress IP."; + }; + }; + + config = lib.mkIf cfg.enable { + users.users."${username}".linger = true; + + # computer-use-linux / cua-driver read the AT-SPI accessibility tree. + services.gnome.at-spi2-core.enable = true; + + # A sparse font set is itself a browser fingerprinting signal, so ship the + # same rich font set the desktops use. + modules.desktop.fonts.enable = true; + + # Mesa userspace (software GL for X11). + hardware.graphics.enable = true; + + # Match the proxy's exit region. + time.timeZone = lib.mkForce timeZone; + + # Browsers keep their profile/preferences in dconf. + programs.dconf.enable = true; + + # Clash Verge provides the proxy. Its core is started by the GUI, which is + # launched inside the session; service mode + TUN mode come from the module. + programs.clash-verge = { + enable = true; + package = pkgs-master.clash-verge-rev; + serviceMode = true; + tunMode = true; + }; + }; +} diff --git a/outputs/x86_64-linux/src/idols-kana.nix b/outputs/x86_64-linux/src/idols-kana.nix index af91df6e..d95e604d 100644 --- a/outputs/x86_64-linux/src/idols-kana.nix +++ b/outputs/x86_64-linux/src/idols-kana.nix @@ -26,6 +26,8 @@ let "secrets/nixos.nix" "modules/nixos/server/server.nix" "modules/nixos/server/qemu-guest-hardware-configuration.nix" + # headless computer-use environment (sway + desktop automation) + "modules/nixos/desktop/computer-use.nix" # host specific "hosts/idols-${name}" ]) diff --git a/outputs/x86_64-linux/src/idols-ruby.nix b/outputs/x86_64-linux/src/idols-ruby.nix index 82f2e537..6c28247b 100644 --- a/outputs/x86_64-linux/src/idols-ruby.nix +++ b/outputs/x86_64-linux/src/idols-ruby.nix @@ -26,6 +26,8 @@ let "secrets/nixos.nix" "modules/nixos/server/server.nix" "modules/nixos/server/qemu-guest-hardware-configuration.nix" + # headless computer-use environment (sway + desktop automation) + "modules/nixos/desktop/computer-use.nix" # host specific "hosts/idols-${name}" ]) diff --git a/outputs/x86_64-linux/tests/computer-use-headless/expected.nix b/outputs/x86_64-linux/tests/computer-use-headless/expected.nix new file mode 100644 index 00000000..2f893ac4 --- /dev/null +++ b/outputs/x86_64-linux/tests/computer-use-headless/expected.nix @@ -0,0 +1,22 @@ +{ + myvars, + lib, +}: +let + hosts = [ + "ruby" + "kana" + ]; +in +lib.genAttrs hosts (name: { + computerUse = true; + atspi = true; + clashVerge = true; + timeZone = "America/Los_Angeles"; + xvfbRestart = "always"; + i3Restart = "always"; + vnc = true; + cuaDriver = name == "ruby"; + brave = true; + chromium = true; +}) diff --git a/outputs/x86_64-linux/tests/computer-use-headless/expr.nix b/outputs/x86_64-linux/tests/computer-use-headless/expr.nix new file mode 100644 index 00000000..186e8d74 --- /dev/null +++ b/outputs/x86_64-linux/tests/computer-use-headless/expr.nix @@ -0,0 +1,31 @@ +{ + myvars, + lib, + outputs, +}: +let + username = myvars.username; + hosts = [ + "ruby" + "kana" + ]; +in +lib.genAttrs hosts ( + name: + let + system = outputs.nixosConfigurations.${name}.config; + home = system.home-manager.users.${username}; + in + { + computerUse = system.modules.desktop.computerUse.enable; + atspi = system.services.gnome.at-spi2-core.enable; + clashVerge = system.programs.clash-verge.enable; + timeZone = system.time.timeZone; + xvfbRestart = home.systemd.user.services.xvfb.Service.Restart; + i3Restart = home.systemd.user.services.i3.Service.Restart; + vnc = home.modules.desktop.computerUse.vnc; + cuaDriver = home.modules.desktop.computerUse.cuaDriver; + brave = home.programs.brave-origin.enable; + chromium = home.programs.chromium.enable; + } +) diff --git a/overlays/computer-use-linux.nix b/overlays/computer-use-linux.nix new file mode 100644 index 00000000..f67d3d12 --- /dev/null +++ b/overlays/computer-use-linux.nix @@ -0,0 +1,3 @@ +_: _final: prev: { + computer-use-linux = prev.callPackage ../pkgs/computer-use-linux { }; +} diff --git a/overlays/cua-driver.nix b/overlays/cua-driver.nix new file mode 100644 index 00000000..1af0d79d --- /dev/null +++ b/overlays/cua-driver.nix @@ -0,0 +1,3 @@ +_: _final: prev: { + cua-driver = prev.callPackage ../pkgs/cua-driver { }; +} diff --git a/pkgs/computer-use-linux/default.nix b/pkgs/computer-use-linux/default.nix new file mode 100644 index 00000000..0612c4ef --- /dev/null +++ b/pkgs/computer-use-linux/default.nix @@ -0,0 +1,56 @@ +{ + lib, + stdenv, + fetchurl, + autoPatchelfHook, +}: +let + version = "0.7.1"; + + # Prebuilt release binaries; the crate is not packaged in nixpkgs. + assets = { + x86_64-linux = { + target = "x86_64-unknown-linux-gnu"; + hash = "sha256-Wp91gQrUKBjIoxIeEsRug4MYgZ6o6bcaDJoqq+RmmbQ="; + }; + aarch64-linux = { + target = "aarch64-unknown-linux-gnu"; + hash = "sha256-e376naC15EyNGfDN3ERTmvCPzd97v7WPzzVYq4zCjps="; + }; + }; + + asset = + assets.${stdenv.hostPlatform.system} + or (throw "computer-use-linux: unsupported system ${stdenv.hostPlatform.system}"); +in +stdenv.mkDerivation { + pname = "computer-use-linux"; + inherit version; + + src = fetchurl { + url = "https://github.com/agent-sh/computer-use-linux/releases/download/v${version}/computer-use-linux-${asset.target}"; + inherit (asset) hash; + }; + + nativeBuildInputs = [ autoPatchelfHook ]; + buildInputs = [ stdenv.cc.cc.lib ]; + + dontUnpack = true; + + installPhase = '' + runHook preInstall + install -Dm755 $src $out/bin/computer-use-linux + runHook postInstall + ''; + + meta = { + description = "MCP server and CLI to control a Linux desktop"; + homepage = "https://github.com/agent-sh/computer-use-linux"; + license = lib.licenses.mit; + platforms = [ + "x86_64-linux" + "aarch64-linux" + ]; + mainProgram = "computer-use-linux"; + }; +} diff --git a/pkgs/cua-driver/default.nix b/pkgs/cua-driver/default.nix new file mode 100644 index 00000000..f6c66165 --- /dev/null +++ b/pkgs/cua-driver/default.nix @@ -0,0 +1,55 @@ +{ + lib, + stdenv, + fetchurl, + autoPatchelfHook, + libxkbcommon, + xorg, +}: +let + version = "0.24.0"; + + # Prebuilt release binary; the crate is not packaged in nixpkgs. + asset = { + url = "https://github.com/trycua/cua/releases/download/cua-driver-rs-v${version}/cua-driver-rs-${version}-linux-x86_64-binary.tar.gz"; + hash = "sha256-s7j/Ullf6xESGaoKyQ47NmGMxoaqggWu1PPn4aZ8e2Q="; + }; +in +stdenv.mkDerivation { + pname = "cua-driver"; + inherit version; + + src = fetchurl asset; + + nativeBuildInputs = [ autoPatchelfHook ]; + buildInputs = [ + stdenv.cc.cc.lib + xorg.libX11 + xorg.libXi + libxkbcommon + ]; + + sourceRoot = "."; + dontConfigure = true; + dontBuild = true; + + installPhase = '' + runHook preInstall + install -Dm755 cua-driver $out/bin/cua-driver + install -Dm755 cua-cursor-theme $out/lib/cua-driver/cua-cursor-theme + install -Dm755 libcua_driver_sdk.so $out/lib/cua-driver/libcua_driver_sdk.so + install -Dm755 cua_driver_node_runtime.node $out/lib/cua-driver/cua_driver_node_runtime.node + install -Dm644 cua_driver_abi.h $out/include/cua-driver/cua_driver_abi.h + mkdir -p $out/share/cua-driver + cp -r wayland-helper $out/share/cua-driver/wayland-helper + runHook postInstall + ''; + + meta = { + description = "Desktop-automation driver for computer-use agents (MCP/CLI)"; + homepage = "https://github.com/trycua/cua"; + license = lib.licenses.mit; + platforms = [ "x86_64-linux" ]; + mainProgram = "cua-driver"; + }; +}