From 00aab5f562fcf56dad1956be00bdfdeb14f927dd Mon Sep 17 00:00:00 2001 From: Ryan Yin Date: Mon, 21 Sep 2026 21:31:27 +0800 Subject: [PATCH] feat(computer-use): add a headless X11/i3 environment Give the idols-ruby and idols-kana VMs a headless computer-use environment: Xvfb plus i3, driven by cua-driver (and computer-use-linux as a fallback). Includes the system module (AT-SPI, fonts, Mesa, Clash Verge proxy, proxy-region timezone), the home i3 module (Xvfb/i3/x11vnc services, browser flags), host wiring and an eval test. --- home/hosts/linux/idols-kana.nix | 10 +- home/hosts/linux/idols-ruby.nix | 12 +- home/linux/gui/i3/default.nix | 175 ++++++++++++++++++ hosts/idols-kana/default.nix | 1 + hosts/idols-ruby/computer-use.nix | 4 + hosts/idols-ruby/default.nix | 1 + modules/nixos/desktop/computer-use.nix | 58 ++++++ outputs/x86_64-linux/src/idols-kana.nix | 2 + outputs/x86_64-linux/src/idols-ruby.nix | 2 + .../tests/computer-use-headless/expected.nix | 22 +++ .../tests/computer-use-headless/expr.nix | 31 ++++ 11 files changed, 316 insertions(+), 2 deletions(-) create mode 100644 home/linux/gui/i3/default.nix create mode 100644 hosts/idols-ruby/computer-use.nix create mode 100644 modules/nixos/desktop/computer-use.nix create mode 100644 outputs/x86_64-linux/tests/computer-use-headless/expected.nix create mode 100644 outputs/x86_64-linux/tests/computer-use-headless/expr.nix diff --git a/home/hosts/linux/idols-kana.nix b/home/hosts/linux/idols-kana.nix index a67103bf..a6da9140 100644 --- a/home/hosts/linux/idols-kana.nix +++ b/home/hosts/linux/idols-kana.nix @@ -1,3 +1,11 @@ { - imports = [ ../../linux/core.nix ]; + imports = [ + ../../linux/core.nix + ../../linux/gui/i3 + ]; + + # Headless X11/i3 session for computer use. + modules.desktop.computerUse.enable = true; + # localhost-only VNC server for manual access (use an SSH tunnel). + modules.desktop.computerUse.vnc = true; } diff --git a/home/hosts/linux/idols-ruby.nix b/home/hosts/linux/idols-ruby.nix index a67103bf..4577dfdf 100644 --- a/home/hosts/linux/idols-ruby.nix +++ b/home/hosts/linux/idols-ruby.nix @@ -1,3 +1,13 @@ { - imports = [ ../../linux/core.nix ]; + imports = [ + ../../linux/core.nix + ../../linux/gui/i3 + ]; + + # Headless X11/i3 session for computer use. + modules.desktop.computerUse.enable = true; + # localhost-only VNC server for manual access (use an SSH tunnel). + modules.desktop.computerUse.vnc = true; + # trycua/cua driver, grey-tested on ruby only. + modules.desktop.computerUse.cuaDriver = true; } diff --git a/home/linux/gui/i3/default.nix b/home/linux/gui/i3/default.nix new file mode 100644 index 00000000..5f453ef7 --- /dev/null +++ b/home/linux/gui/i3/default.nix @@ -0,0 +1,175 @@ +{ + pkgs, + pkgs-x64, + config, + lib, + ... +}: +let + cfg = config.modules.desktop.computerUse; + + display = ":0"; + screen = "1920x1080x24"; + + # Keep this list minimal: the goal is to look like a real user device, not to + # harden the browser. The browser goes out through Clash Verge's proxy, so + # non-proxied WebRTC UDP must stay disabled (a real IP leak would contradict + # the exit region); everything else is a stock-looking setting. + browserFlags = [ + "--force-webrtc-ip-handling-policy=disable_non_proxied_udp" + "--disable-blink-features=AutomationControlled" + "--force-renderer-accessibility" + "--lang=en-US" + "--window-size=1920,1080" + "--window-position=0,0" + "--no-first-run" + "--no-default-browser-check" + ]; + + # Exec'd by i3 once the X session is up. + initScript = pkgs.writeShellScript "computer-use-init" '' + set -eu + + # Make the X session visible to the systemd/dbus user manager, so MCP + # servers and other user services started later inherit DISPLAY. + export DISPLAY=${display} + ${pkgs.dbus}/bin/dbus-update-activation-environment --systemd \ + DISPLAY XDG_SESSION_TYPE XDG_CURRENT_DESKTOP || true + + # This is an X11 session: drop any stale Wayland variables (left over from + # a previous session), otherwise tools such as x11vnc refuse to start. + ${pkgs.systemd}/bin/systemctl --user unset-environment WAYLAND_DISPLAY || true + + # Enable the AT-SPI bridge so computer-use agents can read accessibility + # trees. at-spi-bus-launcher derives org.a11y.Status.IsEnabled from this key. + GSETTINGS_BACKEND=dconf \ + DBUS_SESSION_BUS_ADDRESS="unix:path=''${XDG_RUNTIME_DIR}/bus" \ + ${pkgs.glib.bin}/bin/gsettings set org.gnome.desktop.interface toolkit-accessibility true \ + >/dev/null 2>&1 || true + + ${lib.optionalString cfg.cuaDriver '' + # cua-driver sends content-free telemetry by default; turn it off. + ${pkgs-x64.cua-driver}/bin/cua-driver telemetry disable >/dev/null 2>&1 || true + ''} + ''; +in +{ + options.modules.desktop.computerUse = { + enable = lib.mkEnableOption "headless X11/i3 session for computer use"; + vnc = lib.mkEnableOption "a localhost-only x11vnc server for manual access"; + cuaDriver = lib.mkEnableOption "the trycua/cua driver (cua-driver)"; + }; + + config = lib.mkIf cfg.enable { + home.packages = + with pkgs; + [ + glib # gsettings, to enable the AT-SPI bridge + gsettings-desktop-schemas # the toolkit-accessibility schema + i3 + kitty # terminal (supports X11 and Wayland) + xdotool # XTEST input (fallback / manual use) + xclip # clipboard + xvfb # headless X server + ] + ++ [ + # MCP server / CLI that drives the desktop (see overlays/computer-use-linux.nix). + pkgs-x64.computer-use-linux + ] + ++ lib.optional cfg.vnc pkgs.x11vnc + ++ lib.optional cfg.cuaDriver pkgs-x64.cua-driver; + + # Headless X server. + systemd.user.services.xvfb = { + Unit = { + Description = "Xvfb headless X server"; + }; + Install.WantedBy = [ "default.target" ]; + Service = { + Type = "simple"; + ExecStart = "${pkgs.xvfb}/bin/Xvfb ${display} -screen 0 ${screen} -nolisten tcp"; + Restart = "always"; + RestartSec = 2; + }; + }; + + # Window manager. + systemd.user.services.i3 = { + Unit = { + Description = "i3 window manager (headless X)"; + After = [ "xvfb.service" ]; + Requires = [ "xvfb.service" ]; + }; + Install.WantedBy = [ "default.target" ]; + Service = { + Type = "simple"; + Environment = [ + "DISPLAY=${display}" + "XDG_SESSION_TYPE=x11" + "XDG_CURRENT_DESKTOP=i3" + ]; + ExecStart = "${pkgs.i3}/bin/i3"; + Restart = "always"; + RestartSec = 2; + }; + }; + + # Optional VNC server for manual access; localhost only, use an SSH tunnel. + systemd.user.services.x11vnc = lib.mkIf cfg.vnc { + Unit = { + Description = "x11vnc (localhost) for manual access"; + After = [ "xvfb.service" ]; + Wants = [ "xvfb.service" ]; + }; + Install.WantedBy = [ "default.target" ]; + Service = { + Type = "simple"; + ExecStart = "${pkgs.x11vnc}/bin/x11vnc -display ${display} -localhost -nopw -forever -shared -rfbport 5900"; + Restart = "always"; + RestartSec = 2; + }; + }; + + # cua-driver daemon, so `cua-driver mcp` / `call` can reach it. + systemd.user.services.cua-driver = lib.mkIf cfg.cuaDriver { + Unit = { + Description = "cua-driver daemon"; + After = [ + "xvfb.service" + "i3.service" + ]; + Wants = [ "xvfb.service" ]; + }; + Install.WantedBy = [ "default.target" ]; + Service = { + Type = "simple"; + Environment = [ + "DISPLAY=${display}" + "XDG_SESSION_TYPE=x11" + ]; + ExecStart = "${pkgs-x64.cua-driver}/bin/cua-driver serve"; + Restart = "always"; + RestartSec = 2; + }; + }; + + xdg.configFile."i3/config".text = '' + # i3 configuration for the headless computer-use session. + # Keep it minimal; see README.md in this directory for the rationale. + + font pango:monospace 10 + default_border normal + + exec --no-startup-id ${initScript} + ''; + + programs.brave-origin = { + enable = true; + commandLineArgs = browserFlags; + }; + programs.chromium = { + enable = true; + commandLineArgs = browserFlags; + }; + }; +} diff --git a/hosts/idols-kana/default.nix b/hosts/idols-kana/default.nix index 0df9270c..593de470 100644 --- a/hosts/idols-kana/default.nix +++ b/hosts/idols-kana/default.nix @@ -18,6 +18,7 @@ in { imports = [ ../idols-ruby/packages.nix + ../idols-ruby/computer-use.nix ../idols-ruby/oci-containers ]; diff --git a/hosts/idols-ruby/computer-use.nix b/hosts/idols-ruby/computer-use.nix new file mode 100644 index 00000000..fab73e0f --- /dev/null +++ b/hosts/idols-ruby/computer-use.nix @@ -0,0 +1,4 @@ +{ + # Headless computer-use environment (sway + desktop automation + browser). + modules.desktop.computerUse.enable = true; +} diff --git a/hosts/idols-ruby/default.nix b/hosts/idols-ruby/default.nix index 2ceca538..cded3243 100644 --- a/hosts/idols-ruby/default.nix +++ b/hosts/idols-ruby/default.nix @@ -18,6 +18,7 @@ in { imports = [ ./packages.nix + ./computer-use.nix ./oci-containers ]; diff --git a/modules/nixos/desktop/computer-use.nix b/modules/nixos/desktop/computer-use.nix new file mode 100644 index 00000000..43d22024 --- /dev/null +++ b/modules/nixos/desktop/computer-use.nix @@ -0,0 +1,58 @@ +{ + config, + lib, + pkgs, + pkgs-master, + myvars, + ... +}: +let + cfg = config.modules.desktop.computerUse; + username = myvars.username; + + # Keep the guest timezone consistent with the proxy's exit region. A + # China-based timezone with a US/JP exit IP is a strong automation signal. + timeZone = if cfg.proxyRegion == "jp" then "Asia/Tokyo" else "America/Los_Angeles"; +in +{ + options.modules.desktop.computerUse = { + enable = lib.mkEnableOption "headless computer-use environment (X11 + desktop automation)"; + proxyRegion = lib.mkOption { + type = lib.types.enum [ + "us" + "jp" + ]; + default = "us"; + description = "Proxy exit region; keeps the timezone consistent with the egress IP."; + }; + }; + + config = lib.mkIf cfg.enable { + users.users."${username}".linger = true; + + # computer-use-linux / cua-driver read the AT-SPI accessibility tree. + services.gnome.at-spi2-core.enable = true; + + # A sparse font set is itself a browser fingerprinting signal, so ship the + # same rich font set the desktops use. + modules.desktop.fonts.enable = true; + + # Mesa userspace (software GL for X11). + hardware.graphics.enable = true; + + # Match the proxy's exit region. + time.timeZone = lib.mkForce timeZone; + + # Browsers keep their profile/preferences in dconf. + programs.dconf.enable = true; + + # Clash Verge provides the proxy. Its core is started by the GUI, which is + # launched inside the session; service mode + TUN mode come from the module. + programs.clash-verge = { + enable = true; + package = pkgs-master.clash-verge-rev; + serviceMode = true; + tunMode = true; + }; + }; +} diff --git a/outputs/x86_64-linux/src/idols-kana.nix b/outputs/x86_64-linux/src/idols-kana.nix index af91df6e..d95e604d 100644 --- a/outputs/x86_64-linux/src/idols-kana.nix +++ b/outputs/x86_64-linux/src/idols-kana.nix @@ -26,6 +26,8 @@ let "secrets/nixos.nix" "modules/nixos/server/server.nix" "modules/nixos/server/qemu-guest-hardware-configuration.nix" + # headless computer-use environment (sway + desktop automation) + "modules/nixos/desktop/computer-use.nix" # host specific "hosts/idols-${name}" ]) diff --git a/outputs/x86_64-linux/src/idols-ruby.nix b/outputs/x86_64-linux/src/idols-ruby.nix index 82f2e537..6c28247b 100644 --- a/outputs/x86_64-linux/src/idols-ruby.nix +++ b/outputs/x86_64-linux/src/idols-ruby.nix @@ -26,6 +26,8 @@ let "secrets/nixos.nix" "modules/nixos/server/server.nix" "modules/nixos/server/qemu-guest-hardware-configuration.nix" + # headless computer-use environment (sway + desktop automation) + "modules/nixos/desktop/computer-use.nix" # host specific "hosts/idols-${name}" ]) diff --git a/outputs/x86_64-linux/tests/computer-use-headless/expected.nix b/outputs/x86_64-linux/tests/computer-use-headless/expected.nix new file mode 100644 index 00000000..2f893ac4 --- /dev/null +++ b/outputs/x86_64-linux/tests/computer-use-headless/expected.nix @@ -0,0 +1,22 @@ +{ + myvars, + lib, +}: +let + hosts = [ + "ruby" + "kana" + ]; +in +lib.genAttrs hosts (name: { + computerUse = true; + atspi = true; + clashVerge = true; + timeZone = "America/Los_Angeles"; + xvfbRestart = "always"; + i3Restart = "always"; + vnc = true; + cuaDriver = name == "ruby"; + brave = true; + chromium = true; +}) diff --git a/outputs/x86_64-linux/tests/computer-use-headless/expr.nix b/outputs/x86_64-linux/tests/computer-use-headless/expr.nix new file mode 100644 index 00000000..186e8d74 --- /dev/null +++ b/outputs/x86_64-linux/tests/computer-use-headless/expr.nix @@ -0,0 +1,31 @@ +{ + myvars, + lib, + outputs, +}: +let + username = myvars.username; + hosts = [ + "ruby" + "kana" + ]; +in +lib.genAttrs hosts ( + name: + let + system = outputs.nixosConfigurations.${name}.config; + home = system.home-manager.users.${username}; + in + { + computerUse = system.modules.desktop.computerUse.enable; + atspi = system.services.gnome.at-spi2-core.enable; + clashVerge = system.programs.clash-verge.enable; + timeZone = system.time.timeZone; + xvfbRestart = home.systemd.user.services.xvfb.Service.Restart; + i3Restart = home.systemd.user.services.i3.Service.Restart; + vnc = home.modules.desktop.computerUse.vnc; + cuaDriver = home.modules.desktop.computerUse.cuaDriver; + brave = home.programs.brave-origin.enable; + chromium = home.programs.chromium.enable; + } +)