Merge pull request #277 from ryan4yin/feat/cua-headless-sway

feat(computer-use): headless X11/i3 + cua-driver on ruby and kana

home/README.md

@@ -22,6 +22,7 @@ home/
 ├── linux/             # Linux-specific home manager configurations
 │   ├── base/          # Linux base configurations
 │   ├── gui/           # Linux GUI applications
+│   │   ├── i3/        # i3 — headless computer-use session (see i3/README.md)
 │   │   ├── niri/      # Niri window manager
 │   │   └── ...
 │   ├── editors/       # Linux-specific editors
@@ -42,6 +43,7 @@ home/
 
 2. **linux**: Linux-specific configuration
    - Desktop environments (Noctalia Shell, Niri compositor)
+   - Headless computer-use session (`gui/i3`, Xvfb + i3; see `gui/i3/README.md`)
    - Linux-specific GUI applications
    - System integration tools
 

home/hosts/linux/idols-kana.nix

@@ -1,3 +1,11 @@
 {
-  imports = [ ../../linux/core.nix ];
+  imports = [
+    ../../linux/core.nix
+    ../../linux/gui/i3
+  ];
+
+  # Headless X11/i3 session for computer use.
+  modules.desktop.computerUse.enable = true;
+  # localhost-only VNC server for manual access (use an SSH tunnel).
+  modules.desktop.computerUse.vnc = true;
 }

home/hosts/linux/idols-ruby.nix

@@ -1,3 +1,13 @@
 {
-  imports = [ ../../linux/core.nix ];
+  imports = [
+    ../../linux/core.nix
+    ../../linux/gui/i3
+  ];
+
+  # Headless X11/i3 session for computer use.
+  modules.desktop.computerUse.enable = true;
+  # localhost-only VNC server for manual access (use an SSH tunnel).
+  modules.desktop.computerUse.vnc = true;
+  # trycua/cua driver, grey-tested on ruby only.
+  modules.desktop.computerUse.cuaDriver = true;
 }

home/linux/gui/i3/README.md

@@ -0,0 +1,39 @@
+# i3 — headless computer-use session
+
+This module provides the headless X11 session used by the computer-use VMs (`idols-ruby`,
+`idols-kana`). It is **not** a normal desktop configuration and is not meant to be used
+interactively by a person.
+
+## Purpose
+
+Give a computer-use agent a virtual 1920x1080 X display it can observe and control: screenshots,
+window discovery/activation, synthetic input, and the AT-SPI accessibility tree. The agent itself
+runs elsewhere and connects over MCP; this module is only the "computer" side.
+
+The session is built on Xvfb plus i3 because the drivers support native X11 at their highest tier
+(cua-driver "Linux X11 — Supported"; computer-use-linux has an i3 backend). cua-driver's Sway lane
+is XWayland/X11-based, so a native X11 session is simpler and better supported than Sway/Wayland.
+
+## How it is wired
+
+- `home/linux/gui/i3/default.nix` defines the option `modules.desktop.computerUse` and the user
+  services: `xvfb` → `i3` → (`x11vnc`, `cua-driver serve`).
+- `modules/nixos/desktop/computer-use.nix` provides the system side (`linger`, AT-SPI, fonts, Mesa,
+  Clash Verge proxy, proxy-region timezone).
+- `computer-use-init.sh` (exec'd by i3) publishes `DISPLAY`/`XDG_SESSION_TYPE` to the systemd user
+  manager, clears any stale `WAYLAND_DISPLAY`, and enables the AT-SPI bridge via a gsettings key.
+- Drivers live in `overlays/cua-driver.nix` and `overlays/computer-use-linux.nix`.
+- Host wiring: `home/hosts/linux/idols-ruby.nix`, `home/hosts/linux/idols-kana.nix`.
+
+## The i3 config
+
+The generated config is intentionally minimal:
+
+- `font pango:monospace 10`
+- `default_border normal` — keeps title bars, so the client area is slightly smaller than the
+  screen. That matches a real desktop and avoids a full-screen-sized viewport.
+- `exec --no-startup-id .../computer-use-init.sh` — one-time session setup, see above.
+
+There are no keybindings, bars or autostarted apps: agents drive the session through the drivers,
+not the keyboard. Manual inspection is done over VNC (`x11vnc`, bound to `127.0.0.1`; reach it
+through an SSH tunnel).

home/linux/gui/i3/default.nix

@@ -0,0 +1,175 @@
+{
+  pkgs,
+  pkgs-x64,
+  config,
+  lib,
+  ...
+}:
+let
+  cfg = config.modules.desktop.computerUse;
+
+  display = ":0";
+  screen = "1920x1080x24";
+
+  # Keep this list minimal: the goal is to look like a real user device, not to
+  # harden the browser. The browser goes out through Clash Verge's proxy, so
+  # non-proxied WebRTC UDP must stay disabled (a real IP leak would contradict
+  # the exit region); everything else is a stock-looking setting.
+  browserFlags = [
+    "--force-webrtc-ip-handling-policy=disable_non_proxied_udp"
+    "--disable-blink-features=AutomationControlled"
+    "--force-renderer-accessibility"
+    "--lang=en-US"
+    "--window-size=1920,1080"
+    "--window-position=0,0"
+    "--no-first-run"
+    "--no-default-browser-check"
+  ];
+
+  # Exec'd by i3 once the X session is up.
+  initScript = pkgs.writeShellScript "computer-use-init" ''
+    set -eu
+
+    # Make the X session visible to the systemd/dbus user manager, so MCP
+    # servers and other user services started later inherit DISPLAY.
+    export DISPLAY=${display}
+    ${pkgs.dbus}/bin/dbus-update-activation-environment --systemd \
+      DISPLAY XDG_SESSION_TYPE XDG_CURRENT_DESKTOP || true
+
+    # This is an X11 session: drop any stale Wayland variables (left over from
+    # a previous session), otherwise tools such as x11vnc refuse to start.
+    ${pkgs.systemd}/bin/systemctl --user unset-environment WAYLAND_DISPLAY || true
+
+    # Enable the AT-SPI bridge so computer-use agents can read accessibility
+    # trees. at-spi-bus-launcher derives org.a11y.Status.IsEnabled from this key.
+    GSETTINGS_BACKEND=dconf \
+      DBUS_SESSION_BUS_ADDRESS="unix:path=''${XDG_RUNTIME_DIR}/bus" \
+      ${pkgs.glib.bin}/bin/gsettings set org.gnome.desktop.interface toolkit-accessibility true \
+      >/dev/null 2>&1 || true
+
+    ${lib.optionalString cfg.cuaDriver ''
+      # cua-driver sends content-free telemetry by default; turn it off.
+      ${pkgs-x64.cua-driver}/bin/cua-driver telemetry disable >/dev/null 2>&1 || true
+    ''}
+  '';
+in
+{
+  options.modules.desktop.computerUse = {
+    enable = lib.mkEnableOption "headless X11/i3 session for computer use";
+    vnc = lib.mkEnableOption "a localhost-only x11vnc server for manual access";
+    cuaDriver = lib.mkEnableOption "the trycua/cua driver (cua-driver)";
+  };
+
+  config = lib.mkIf cfg.enable {
+    home.packages =
+      with pkgs;
+      [
+        glib # gsettings, to enable the AT-SPI bridge
+        gsettings-desktop-schemas # the toolkit-accessibility schema
+        i3
+        kitty # terminal (supports X11 and Wayland)
+        xdotool # XTEST input (fallback / manual use)
+        xclip # clipboard
+        xvfb # headless X server
+      ]
+      ++ [
+        # MCP server / CLI that drives the desktop (see overlays/computer-use-linux.nix).
+        pkgs-x64.computer-use-linux
+      ]
+      ++ lib.optional cfg.vnc pkgs.x11vnc
+      ++ lib.optional cfg.cuaDriver pkgs-x64.cua-driver;
+
+    # Headless X server.
+    systemd.user.services.xvfb = {
+      Unit = {
+        Description = "Xvfb headless X server";
+      };
+      Install.WantedBy = [ "default.target" ];
+      Service = {
+        Type = "simple";
+        ExecStart = "${pkgs.xvfb}/bin/Xvfb ${display} -screen 0 ${screen} -nolisten tcp";
+        Restart = "always";
+        RestartSec = 2;
+      };
+    };
+
+    # Window manager.
+    systemd.user.services.i3 = {
+      Unit = {
+        Description = "i3 window manager (headless X)";
+        After = [ "xvfb.service" ];
+        Requires = [ "xvfb.service" ];
+      };
+      Install.WantedBy = [ "default.target" ];
+      Service = {
+        Type = "simple";
+        Environment = [
+          "DISPLAY=${display}"
+          "XDG_SESSION_TYPE=x11"
+          "XDG_CURRENT_DESKTOP=i3"
+        ];
+        ExecStart = "${pkgs.i3}/bin/i3";
+        Restart = "always";
+        RestartSec = 2;
+      };
+    };
+
+    # Optional VNC server for manual access; localhost only, use an SSH tunnel.
+    systemd.user.services.x11vnc = lib.mkIf cfg.vnc {
+      Unit = {
+        Description = "x11vnc (localhost) for manual access";
+        After = [ "xvfb.service" ];
+        Wants = [ "xvfb.service" ];
+      };
+      Install.WantedBy = [ "default.target" ];
+      Service = {
+        Type = "simple";
+        ExecStart = "${pkgs.x11vnc}/bin/x11vnc -display ${display} -localhost -nopw -forever -shared -rfbport 5900";
+        Restart = "always";
+        RestartSec = 2;
+      };
+    };
+
+    # cua-driver daemon, so `cua-driver mcp` / `call` can reach it.
+    systemd.user.services.cua-driver = lib.mkIf cfg.cuaDriver {
+      Unit = {
+        Description = "cua-driver daemon";
+        After = [
+          "xvfb.service"
+          "i3.service"
+        ];
+        Wants = [ "xvfb.service" ];
+      };
+      Install.WantedBy = [ "default.target" ];
+      Service = {
+        Type = "simple";
+        Environment = [
+          "DISPLAY=${display}"
+          "XDG_SESSION_TYPE=x11"
+        ];
+        ExecStart = "${pkgs-x64.cua-driver}/bin/cua-driver serve";
+        Restart = "always";
+        RestartSec = 2;
+      };
+    };
+
+    xdg.configFile."i3/config".text = ''
+      # i3 configuration for the headless computer-use session.
+      # Keep it minimal; see README.md in this directory for the rationale.
+
+      font pango:monospace 10
+      default_border normal
+
+      exec --no-startup-id ${initScript}
+    '';
+
+    programs.brave-origin = {
+      enable = true;
+      commandLineArgs = browserFlags;
+    };
+    programs.chromium = {
+      enable = true;
+      commandLineArgs = browserFlags;
+    };
+  };
+}

hosts/idols-kana/default.nix

@@ -18,6 +18,7 @@ in
 {
   imports = [
     ../idols-ruby/packages.nix
+    ../idols-ruby/computer-use.nix
     ../idols-ruby/oci-containers
   ];
 

hosts/idols-ruby/computer-use.nix

@@ -0,0 +1,4 @@
+{
+  # Headless computer-use environment (sway + desktop automation + browser).
+  modules.desktop.computerUse.enable = true;
+}

hosts/idols-ruby/default.nix

@@ -18,6 +18,7 @@ in
 {
   imports = [
     ./packages.nix
+    ./computer-use.nix
     ./oci-containers
   ];
 

modules/nixos/desktop/computer-use.nix

@@ -0,0 +1,58 @@
+{
+  config,
+  lib,
+  pkgs,
+  pkgs-master,
+  myvars,
+  ...
+}:
+let
+  cfg = config.modules.desktop.computerUse;
+  username = myvars.username;
+
+  # Keep the guest timezone consistent with the proxy's exit region. A
+  # China-based timezone with a US/JP exit IP is a strong automation signal.
+  timeZone = if cfg.proxyRegion == "jp" then "Asia/Tokyo" else "America/Los_Angeles";
+in
+{
+  options.modules.desktop.computerUse = {
+    enable = lib.mkEnableOption "headless computer-use environment (X11 + desktop automation)";
+    proxyRegion = lib.mkOption {
+      type = lib.types.enum [
+        "us"
+        "jp"
+      ];
+      default = "us";
+      description = "Proxy exit region; keeps the timezone consistent with the egress IP.";
+    };
+  };
+
+  config = lib.mkIf cfg.enable {
+    users.users."${username}".linger = true;
+
+    # computer-use-linux / cua-driver read the AT-SPI accessibility tree.
+    services.gnome.at-spi2-core.enable = true;
+
+    # A sparse font set is itself a browser fingerprinting signal, so ship the
+    # same rich font set the desktops use.
+    modules.desktop.fonts.enable = true;
+
+    # Mesa userspace (software GL for X11).
+    hardware.graphics.enable = true;
+
+    # Match the proxy's exit region.
+    time.timeZone = lib.mkForce timeZone;
+
+    # Browsers keep their profile/preferences in dconf.
+    programs.dconf.enable = true;
+
+    # Clash Verge provides the proxy. Its core is started by the GUI, which is
+    # launched inside the session; service mode + TUN mode come from the module.
+    programs.clash-verge = {
+      enable = true;
+      package = pkgs-master.clash-verge-rev;
+      serviceMode = true;
+      tunMode = true;
+    };
+  };
+}

outputs/x86_64-linux/src/idols-kana.nix

@@ -26,6 +26,8 @@ let
         "secrets/nixos.nix"
         "modules/nixos/server/server.nix"
         "modules/nixos/server/qemu-guest-hardware-configuration.nix"
+        # headless computer-use environment (sway + desktop automation)
+        "modules/nixos/desktop/computer-use.nix"
         # host specific
         "hosts/idols-${name}"
       ])

outputs/x86_64-linux/src/idols-ruby.nix

@@ -26,6 +26,8 @@ let
         "secrets/nixos.nix"
         "modules/nixos/server/server.nix"
         "modules/nixos/server/qemu-guest-hardware-configuration.nix"
+        # headless computer-use environment (sway + desktop automation)
+        "modules/nixos/desktop/computer-use.nix"
         # host specific
         "hosts/idols-${name}"
       ])

outputs/x86_64-linux/tests/computer-use-headless/expected.nix

@@ -0,0 +1,22 @@
+{
+  myvars,
+  lib,
+}:
+let
+  hosts = [
+    "ruby"
+    "kana"
+  ];
+in
+lib.genAttrs hosts (name: {
+  computerUse = true;
+  atspi = true;
+  clashVerge = true;
+  timeZone = "America/Los_Angeles";
+  xvfbRestart = "always";
+  i3Restart = "always";
+  vnc = true;
+  cuaDriver = name == "ruby";
+  brave = true;
+  chromium = true;
+})

outputs/x86_64-linux/tests/computer-use-headless/expr.nix

@@ -0,0 +1,31 @@
+{
+  myvars,
+  lib,
+  outputs,
+}:
+let
+  username = myvars.username;
+  hosts = [
+    "ruby"
+    "kana"
+  ];
+in
+lib.genAttrs hosts (
+  name:
+  let
+    system = outputs.nixosConfigurations.${name}.config;
+    home = system.home-manager.users.${username};
+  in
+  {
+    computerUse = system.modules.desktop.computerUse.enable;
+    atspi = system.services.gnome.at-spi2-core.enable;
+    clashVerge = system.programs.clash-verge.enable;
+    timeZone = system.time.timeZone;
+    xvfbRestart = home.systemd.user.services.xvfb.Service.Restart;
+    i3Restart = home.systemd.user.services.i3.Service.Restart;
+    vnc = home.modules.desktop.computerUse.vnc;
+    cuaDriver = home.modules.desktop.computerUse.cuaDriver;
+    brave = home.programs.brave-origin.enable;
+    chromium = home.programs.chromium.enable;
+  }
+)

overlays/computer-use-linux.nix

@@ -0,0 +1,3 @@
+_: _final: prev: {
+  computer-use-linux = prev.callPackage ../pkgs/computer-use-linux { };
+}

overlays/cua-driver.nix

@@ -0,0 +1,3 @@
+_: _final: prev: {
+  cua-driver = prev.callPackage ../pkgs/cua-driver { };
+}

pkgs/computer-use-linux/default.nix

@@ -0,0 +1,56 @@
+{
+  lib,
+  stdenv,
+  fetchurl,
+  autoPatchelfHook,
+}:
+let
+  version = "0.7.1";
+
+  # Prebuilt release binaries; the crate is not packaged in nixpkgs.
+  assets = {
+    x86_64-linux = {
+      target = "x86_64-unknown-linux-gnu";
+      hash = "sha256-Wp91gQrUKBjIoxIeEsRug4MYgZ6o6bcaDJoqq+RmmbQ=";
+    };
+    aarch64-linux = {
+      target = "aarch64-unknown-linux-gnu";
+      hash = "sha256-e376naC15EyNGfDN3ERTmvCPzd97v7WPzzVYq4zCjps=";
+    };
+  };
+
+  asset =
+    assets.${stdenv.hostPlatform.system}
+      or (throw "computer-use-linux: unsupported system ${stdenv.hostPlatform.system}");
+in
+stdenv.mkDerivation {
+  pname = "computer-use-linux";
+  inherit version;
+
+  src = fetchurl {
+    url = "https://github.com/agent-sh/computer-use-linux/releases/download/v${version}/computer-use-linux-${asset.target}";
+    inherit (asset) hash;
+  };
+
+  nativeBuildInputs = [ autoPatchelfHook ];
+  buildInputs = [ stdenv.cc.cc.lib ];
+
+  dontUnpack = true;
+
+  installPhase = ''
+    runHook preInstall
+    install -Dm755 $src $out/bin/computer-use-linux
+    runHook postInstall
+  '';
+
+  meta = {
+    description = "MCP server and CLI to control a Linux desktop";
+    homepage = "https://github.com/agent-sh/computer-use-linux";
+    license = lib.licenses.mit;
+    platforms = [
+      "x86_64-linux"
+      "aarch64-linux"
+    ];
+    mainProgram = "computer-use-linux";
+  };
+}

pkgs/cua-driver/default.nix

@@ -0,0 +1,55 @@
+{
+  lib,
+  stdenv,
+  fetchurl,
+  autoPatchelfHook,
+  libxkbcommon,
+  xorg,
+}:
+let
+  version = "0.24.0";
+
+  # Prebuilt release binary; the crate is not packaged in nixpkgs.
+  asset = {
+    url = "https://github.com/trycua/cua/releases/download/cua-driver-rs-v${version}/cua-driver-rs-${version}-linux-x86_64-binary.tar.gz";
+    hash = "sha256-s7j/Ullf6xESGaoKyQ47NmGMxoaqggWu1PPn4aZ8e2Q=";
+  };
+in
+stdenv.mkDerivation {
+  pname = "cua-driver";
+  inherit version;
+
+  src = fetchurl asset;
+
+  nativeBuildInputs = [ autoPatchelfHook ];
+  buildInputs = [
+    stdenv.cc.cc.lib
+    xorg.libX11
+    xorg.libXi
+    libxkbcommon
+  ];
+
+  sourceRoot = ".";
+  dontConfigure = true;
+  dontBuild = true;
+
+  installPhase = ''
+    runHook preInstall
+    install -Dm755 cua-driver $out/bin/cua-driver
+    install -Dm755 cua-cursor-theme $out/lib/cua-driver/cua-cursor-theme
+    install -Dm755 libcua_driver_sdk.so $out/lib/cua-driver/libcua_driver_sdk.so
+    install -Dm755 cua_driver_node_runtime.node $out/lib/cua-driver/cua_driver_node_runtime.node
+    install -Dm644 cua_driver_abi.h $out/include/cua-driver/cua_driver_abi.h
+    mkdir -p $out/share/cua-driver
+    cp -r wayland-helper $out/share/cua-driver/wayland-helper
+    runHook postInstall
+  '';
+
+  meta = {
+    description = "Desktop-automation driver for computer-use agents (MCP/CLI)";
+    homepage = "https://github.com/trycua/cua";
+    license = lib.licenses.mit;
+    platforms = [ "x86_64-linux" ];
+    mainProgram = "cua-driver";
+  };
+}