diff --git a/.github/workflows/shared-desktop.yml b/.github/workflows/shared-desktop.yml new file mode 100644 index 0000000..aa2e52e --- /dev/null +++ b/.github/workflows/shared-desktop.yml @@ -0,0 +1,42 @@ +name: Shared desktop verification +on: + pull_request: + push: + branches: [main] + workflow_dispatch: +permissions: + contents: read +concurrency: + group: shared-desktop-${{ github.ref }} + cancel-in-progress: true +jobs: + verify: + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-15] + runs-on: ${{ matrix.os }} + timeout-minutes: 20 + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - uses: oven-sh/setup-bun@v2 + with: + bun-version: 1.3.10 + - uses: actions/setup-node@v4 + with: + node-version: '24' + - run: bun install --frozen-lockfile + - run: bun run check + - run: bun run build + - name: Compile permission-owning Mac helper + if: runner.os == 'macOS' + env: + CU_NATIVE_DIR: ${{ runner.temp }}/opcode-native + run: | + bash scripts/build-native.sh + bun test tests/mac-lock.test.ts + node scripts/test-macos-desktop.mjs + bun scripts/test-macos-desktop-live.ts + - run: bun test diff --git a/CHANGELOG.md b/CHANGELOG.md index 5e3f00a..63a24a4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,14 @@ # Changelog +## Unreleased + +- Add a macOS backend for the shared desktop broker through the permission-owning Opcode helper. +- Add scoped multiplayer presence, named cursor overlays, participant lists, and explicit input handoff for people and agents. +- Add private sharing credentials, HTTPS clients, and OpenSSH tunnels with verified forwarding readiness. +- Publish agent presence automatically through MCP; preserve uncertain input outcomes and held-input recovery across failures. +- Add remote-access, multiplayer HTTP/browser, and Mac transport tests, plus cross-platform verification CI. + + ## 0.3.0 — 2026-10-03 - Add a shared Linux X11 desktop service with full-display and window capture, input, and attachment to existing displays. diff --git a/README.md b/README.md index 7415d2e..2416d50 100644 --- a/README.md +++ b/README.md @@ -118,9 +118,11 @@ A new session is another way to refresh discovery. The skill teaches the agent t find or launch the right app, choose exact controls, verify results, and recover from local blockers while retaining ownership of the full workflow. You should not have to dictate each click. -**A cloud coding workspace needs a Mac command bridge supplied by its host.** -Install and execute Jev on the Mac through that bridge. A Linux shell or cloud MCP -process alone cannot operate your Mac. Jev does not provide a remote bridge. +**A cloud coding workspace can use the scoped `cu desktop-api` SSH/HTTPS bridge** +to operate an existing Mac desktop; see [remote desktops and multiplayer](#remote-desktops-and-multiplayer). +The Mac must run the broker and permission-owning helper. The semantic `cu observe` +and `cu execute` commands above still run on the Mac, through shell access or a +command bridge supplied by the host. ## Optional delegated workflows @@ -186,6 +188,23 @@ Bun manages packages and runs the CLI. Effect manages native driver requests. Reference repositories live in ignored `.repos/`; dependencies, build output, secrets, and `.context/` artifacts are also ignored. +## Remote desktops and multiplayer + +Share an existing Mac or Linux X11 desktop with people and agents. Each +participant has a named visual cursor; a scoped input lease controls who can +click or type. Takeover fences previous input, and expired or revoked access +removes that participant. Existing signed-in apps stay on the target computer. + +Use `cu desktop-api attach`, `share`, `tunnel`, `viewer`, and `mcp` from the npm +CLI. Remote access uses HTTPS or OpenSSH, including your existing Tailnet SSH +aliases. See [Mac setup and limits](docs/MAC_DESKTOP.md) and the +[shared desktop contract](docs/DESKTOP_SERVICE.md). The standalone legacy Mac +bundle does not include these `desktop-api` commands; use the npm distribution. + +Cursors are multiplayer overlays. OS input has one owner at a time; independent +simultaneous physical cursors and VM management are not implemented. The viewer +uses bounded PNG polling rather than video streaming. + ## Linux cloud desktops Attach to an existing provider X11 desktop with raw screenshots, shared control ownership, authenticated viewing and headless recording. No detector or model API key is required. Start with the [Zuse integration guide](docs/ZUSE_INTEGRATION.md), then the [shared desktop service contract](docs/DESKTOP_SERVICE.md). Creating a separate Xvfb desktop remains available for local isolation. diff --git a/docs/DESKTOP_SERVICE.md b/docs/DESKTOP_SERVICE.md index 519381d..1f9f20b 100644 --- a/docs/DESKTOP_SERVICE.md +++ b/docs/DESKTOP_SERVICE.md @@ -2,10 +2,11 @@ Opcode owns capture, input arbitration, scoped local credentials and recording. The host owns workspace accounts, provider allocation, ingress and lifecycle. -No reasoning model or model key is needed. Linux X11 is the backend implemented -here; this is not a Wayland, Windows, audio or multi-monitor implementation. +No reasoning model or model key is needed. Linux X11 and macOS are implemented backends. For Mac installation, permissions, +coordinates and limitations, see [Shared Mac desktop](MAC_DESKTOP.md). This is +not a Wayland, Windows, audio or multi-monitor desktop implementation. -## Install and attach +## Linux install and attach ```sh npm install --omit=optional @opcodehq/cu @@ -84,37 +85,45 @@ Actions: `click`, `doubleClick`, `rightClick`, `hover` with `x,y`; `drag` adds `toX,toY`; `scroll` has `amount` −30…30 and `axis` x/y; `text` has UTF-8 `text` (up to 8000 characters), with optional `pasteKey: "Control+Shift+v"` or `"Shift+Insert"` for terminals (known terminal classes select the matching shortcut automatically; other apps use Control+v); `key` uses names/combinations such as `Control+a`; `keyDown`/`keyUp` hold one key; `buttonDown`/`buttonUp` take button 1–3; -`focus` takes `windowId`; `launch` takes an installed `.desktop` `appId`. -Launching uses the desktop entry through `gio`, not caller-supplied shell text. +`focus` takes `windowId`. On Linux, `launch` takes an installed `.desktop` +`appId` and uses its desktop entry through `gio`, not caller-supplied shell text. +On macOS, `appId` is an installed application bundle ID. `pasteKey` applies only +to Linux; Mac text uses Unicode events. Coordinates are pixels in the delivered image, with explicit identity scale and root offsets in `imageToDesktop`. No implicit crops/downscaling/CSS coordinates. -Full-display input respects keyboard focus. Window-target input raises/focuses -that window, matching the legacy isolated X11 behavior. Text pastes through that -X display's clipboard and replaces its previous selection (Shift+Insert also sets PRIMARY); preservation is not -promised. Verify the resulting app content. +Full-display input respects keyboard focus. On Linux, window-target input +raises/focuses that window, matching the legacy isolated X11 behavior. Text +pastes through that X display's clipboard and replaces its previous selection +(Shift+Insert also sets PRIMARY); preservation is not promised. On macOS, focus +the target window and obtain a fresh observation before physical input. Mac +captures use logical screen points, including on Retina displays; see +[Mac coordinates and limits](MAC_DESKTOP.md#control-and-coordinates). Verify the +resulting app content. Observations are grant-bound, expire after 30 seconds and are consumed by input. Geometry, focus and a sampled pixel-difference check reject changed scenes. This check tolerates small changes but can reject animation and cannot detect every UI change. There is a time-of-check/time-of-use gap; dispatch acknowledgement -is not verified app success. There are at most 16 queued mutations and 32 retained -observations. Duplicate mutation IDs return the last result while present in the +is not verified app success. There are at most 16 queued mutations, eight retained observations per grant, +and 512 observations overall. Observations older than 30 seconds are evicted. Duplicate mutation IDs return the last result while present in the 512-entry cache; reuse with different content is rejected. Never retry uncertain input automatically or promise exactly-once delivery across crashes. -One broker serves all new MCP, CLI and viewer clients. While it is registered, -legacy window input and browser mutations on that display fail closed instead +One broker serves all `desktop-api` MCP, CLI and viewer clients. On Linux, while +it is registered, legacy window input and browser mutations on that display fail closed instead of opening a second control path. Existing window/browser workflows still work on displays without a broker. The native helper and arbitrary X clients remain -outside this application-level trust boundary. +outside this application-level trust boundary. Mac semantic sessions and local +hardware input also remain outside the shared broker; see the +[Mac control boundaries](MAC_DESKTOP.md#boundaries-and-recovery). ## Viewing and reverse proxies GET `/view#TOKEN` opens the standalone viewer. Fragment tokens are removed from the address bar and held in sessionStorage; API requests use -Authorization headers. Add `?windowId=ID` before the fragment for window-only viewing. A viewer grant needs `viewer-read`; human control also -needs `input-control`. Observer buttons are hidden and mutations are rejected +Authorization headers. Add `?windowId=ID` before the fragment for window-only viewing. A viewer grant needs `viewer-read`; joining and showing a cursor also need +`presence`, and human control needs `input-control`. Observer buttons are hidden and mutations are rejected server-side. Never use the host master token as a viewer link. For embedding, use `/session` for generation/scopes, `/frame` for a PNG observation, @@ -134,6 +143,54 @@ Keep a stable scoped credential per client session; do not mint a new grant per The proxy must revoke that credential when host authorization is withdrawn. The reference callback is not a replacement for host account authentication. +## Multiplayer and remote clients + +`share` creates a private, expiring credential without printing its token: + +```sh +cu desktop-api share --display :1 --subject alice --role controller --output /private/alice.json +cu desktop-api viewer --credential-file /private/alice.json +cu desktop-api mcp --credential-file /private/alice.json +``` + +Use a separate grant for each person or agent. `viewer` grants watch/presence; +`controller` and `agent` also grant input. Administrative descriptors are refused +by the viewer. The host can revoke each returned grant ID independently. + +Add `--endpoint https://desktop.example.com/desktop` when sharing through your +configured TLS proxy. Remote plaintext endpoints and redirects are rejected. +For SSH, use `tunnel --ssh HOST --remote-credential-file ABSOLUTE_PATH --output +LOCAL_PATH`. It uses existing OpenSSH keys/config and known hosts, waits for a +confirmed forward before sending any token, and deletes its local credential +when closed. The tunnel remains in the foreground; it does not reconnect or +replay input. Configure the broker `--origin http://127.0.0.1:4311` for the +default forwarded viewer port, or match your chosen `--local-port`. The native +broker origin remains permitted. HTTPS ingress uses its exact configured origin. + +The additive `presence` scope admits these methods: + +| Method | Behavior | +| --- | --- | +| `presence.join` | Optional `name` and display-only `role` (`human`/`agent`); returns a participant ID/color. | +| `presence.update` | Owned `participantId`, optional `cursor` (null to hide, otherwise normalized `x,y` and optional `target`, default display); refreshes its heartbeat. | +| `presence.leave` | Removes the owned participant and fences its associated input lease. | +| `presence.list` | Requires `viewer-read`; returns participants and current controller, never lease IDs. | + +Join is bounded to eight tabs per grant and 64 participants per broker. Heartbeats +expire after 15 seconds; update rate is capped at 25/second per participant. +Names are untrusted display text. Roles do not confer permission. `acquire` and +`takeover` accept an optional owned `participantId`; legacy clients continue +working without presence. Revocation, expiry, shutdown and rebind remove presence +and fence associated input. MCP joins as an agent and maintains its heartbeat; +the browser updates cursor position at most ten times per second. `/frame` carries +presence and controller state alongside the image. Window cursors are only shown +in viewers of that same target. + +Actual OS input remains exclusive. Visual cursor updates never inject input. +Takeover waits for previous input cleanup. Failed held-key/button releases remain +journaled and block control until cleanup succeeds, including after restart. +Local hardware and programs outside this broker remain outside arbitration. + ## Recording Display resizing also requires the `xrandr` command and a compatible existing display mode. diff --git a/docs/HARNESS_SETUP.md b/docs/HARNESS_SETUP.md index 2f145df..31064af 100644 --- a/docs/HARNESS_SETUP.md +++ b/docs/HARNESS_SETUP.md @@ -8,6 +8,11 @@ Exact native tools work without a TypeSafe key; Jev is optional. `cu task --provider NAME --model ID` runs full workflows through Vercel AI SDK; `cu providers` lists the built-in provider routes and credential variables. +For shared Mac/Linux desktops, remote SSH/HTTPS access, and multiplayer, use +the npm CLI’s `cu desktop-api` commands. The standalone bundle described below +does not include those commands. See [shared desktop setup](DESKTOP_SERVICE.md) +and [Mac setup](MAC_DESKTOP.md). + ## First installation Install an extracted standalone Mac bundle with its `install.sh`, then: @@ -89,7 +94,7 @@ subscription-backed agent to use native computer tools. | Symptom | Fix | | --- | --- | | Setup exits 2 | Installation succeeded but required readiness checks remain. Follow the printed fixes and rerun `jev doctor`. | -| No Mac / Linux platform | Run through the harness's existing Mac command bridge or use a local Mac harness. Installing locally in the cloud is insufficient. | +| Mac semantic tools from Linux/cloud | Run semantic commands through a Mac shell/command bridge, or use the separate `cu desktop-api` SSH/HTTPS workflow for shared desktop capture and input. The target Mac must run its helper and broker. | | Accessibility missing | Request it from the same command host that will run Jev; grant the host macOS identifies and restart it if requested. | | Key missing after saving | Check the command host's user and `JEV_SETTINGS_PATH`; the app and shell must read the same settings file. | | Capture works in Electron but fails from CLI | Screen Recording grants may differ by host. Grant the actual terminal/agent host for visual tasks. | diff --git a/docs/MAC_DESKTOP.md b/docs/MAC_DESKTOP.md new file mode 100644 index 0000000..d27b244 --- /dev/null +++ b/docs/MAC_DESKTOP.md @@ -0,0 +1,160 @@ +# Shared Mac desktop + +Opcode attaches to the signed-in user's existing Mac desktop. Other people and +agents can observe that desktop, show named cursors, and request exclusive input +control. No VM, application migration, credential copying, or second login is +created. Apps already signed in on the target Mac keep their existing sessions. + +## Prepare the target Mac + +Use macOS 14 or newer with an active graphical login, Node.js 20+, Bun 1.3.10+, +and Xcode Command Line Tools. Install the npm CLI build containing these commands and its +permission-owning helper; the legacy standalone bundle does not contain +`desktop-api`: + +```sh +# Install the package built from this version, then initialize the Mac helper: +cu install +cu doctor +``` + +In System Settings → Privacy & Security, grant **Opcode** Accessibility and Screen +Recording. Follow the installer's restart instructions if macOS requests them. +The broker connects to the installed helper's private, same-user Unix socket; it +does not run a standalone driver under a new permission identity. An older helper +must be updated before it understands the shared desktop methods. + +Start a broker with a lifecycle identifier you retain for this desktop session: + +```sh +cu desktop-api attach --display macos --generation mac-work-session-1 \ + --origin http://127.0.0.1:4311 +cu desktop-api call --display macos --method health +``` + +The explicit origin permits the SSH viewer on client port 4311; use the same port +on each client. For an HTTPS reverse proxy, set its exact origin instead. To change +an existing broker’s origin, shut it down and attach again with new credentials. + +The broker binds to loopback. One broker owns the user's Mac display. Repeating +attach with the same generation reuses it. Use the existing generation when +sharing; changing a generation is a lifecycle change, not a way to steal control. + +## Share with a person or agent + +On the target Mac, create a separate scoped credential for each participant: + +```sh +cu desktop-api share --display macos --subject alice --role controller \ + --output "$HOME/alice-desktop.json" +cu desktop-api share --display macos --subject build-agent --role agent \ + --output "$HOME/build-agent-desktop.json" +``` + +Roles are `viewer`, `controller`, and `agent`. Viewers can watch and show a cursor; +controllers and agents can also request input control. Credentials expire after +one hour by default; `--ttl-ms` sets a shorter lifetime. Keep credential files +private. Never distribute the host registry descriptor or its administrative +token. The share command returns the grant ID for revocation: + +```sh +cu desktop-api call --display macos --method revoke --args '{"id":"GRANT_ID"}' +``` + +For another computer, establish normal SSH authentication and verify the target's +host key first. A Tailnet SSH hostname or an SSH configuration alias works. On the +client computer, install the same npm CLI build (Node.js 20+); remote clients do +not need to install a local native helper. Keep this tunnel command running: + +```sh +cu desktop-api tunnel --ssh user@my-mac \ + --remote-credential-file /Users/user/alice-desktop.json \ + --output "$HOME/alice-tunnel.json" --local-port 4311 +``` + +The absolute remote credential path is on the target Mac. The tunnel fetches only +that scoped credential, forwards the loopback broker through OpenSSH, and deletes +its local credential when it closes. It does not disable SSH host verification or +reconnect and replay desktop input automatically. + +In another client terminal: + +```sh +cu desktop-api viewer --credential-file "$HOME/alice-tunnel.json" +cu desktop-api mcp --credential-file "$HOME/alice-tunnel.json" +``` + +Open the viewer URL in a browser. Its fragment contains a bearer credential, so +treat the URL as private. Use the MCP command as the stdio command in your agent's +MCP configuration. Give each independently controlled participant its own grant. +For HTTPS deployments, pass the externally configured HTTPS broker endpoint to +`share --endpoint`; TLS/reverse-proxy configuration is separate from the broker. + +## Control and coordinates + +Every participant has a named overlay cursor. Moving that cursor does not move +the Mac's system pointer or acquire control. The viewer shows the current input +owner. **Take control** transfers the lease, cancels queued input from the prior +owner, and fences its in-flight operation before the new owner can act. **Stop** +releases broker-held input. Wait for confirmation before intervening. + +Agent tools follow this sequence: acquire a lease, observe the target, submit one +input with that lease and observation ID, and observe again to verify the outcome. +Renew the lease during longer workflows. The MCP connection automatically joins +as an agent, maintains its presence heartbeat, and publishes its proposed pointer +location; CLI clients can use the presence methods explicitly. Stale images, changed focus, changed +geometry, expired grants, and mismatched generations reject input. A lost reply +has unknown delivery; observe again instead of retrying the action. + +Captures are PNGs in logical screen points, one image pixel per point, including +on Retina displays. Input coordinates are relative to the returned image. The +observation includes the target's desktop origin, which can be negative for a +window on another display. Full-display capture selects the primary display. +Use the window list and its IDs to select an exact window. Before physical input +to a window target, focus that window and obtain a fresh observation; otherwise +input is rejected. Before scrolling or pressing a held mouse button on a window, +hover inside that window and observe again; a pointer outside the target is rejected. +Window capture does not provide an isolated input session. + +Supported input includes clicks, double/right clicks, hover, drag, vertical and +horizontal scroll, text, named keys and modifier combinations, held keys/buttons, +window focus, and launching an installed application by bundle ID. Text uses +Unicode events rather than replacing the clipboard. Fields identified by Accessibility as secure password fields +reject remote input; custom-drawn controls may not expose that classification. Keyboard shortcuts use the Mac's physical key map; text entry +is preferable when exact characters matter. Display resizing is unsupported. + +## Boundaries and recovery + +macOS has one physical desktop pointer and foreground keyboard target. Overlay +cursors provide multiplayer presence; independent simultaneous physical cursors +or eight isolated Simulator input streams are not provided. Agents can select +different Simulator windows, but physical actions are serialized through the +single lease. This API does not promise background interaction with arbitrary +apps or keep the user's pointer undisturbed during automation. + +The broker controls only input routed through it. Local hardware input, other +applications, and separate CU semantic sessions can bypass its lease. There is +no OS-wide input lock or automatic takeover on physical mouse movement. Use Stop +or Take control and await confirmation before using the local mouse/keyboard; +do not run another physical-input automation tool on the same desktop. A native +operation already handed to macOS cannot be undone; cancellation waits for its +reply, and unconfirmed delivery remains unknown. + +Screen locks, permission revocation, helper exits, logout, and inaccessible +windows can interrupt capture or input. Fix permissions/login first, restart the +broker if its helper connection was lost, and obtain fresh grants/observations as +required. The service does not reconnect and replay a pending native action. It +persists broker-held key/button state for release on startup. Broker shutdown +does not terminate apps or the installed helper. + +Mac broker ownership uses a private per-user file lock held by its native helper +connection. Disconnects and crashes release the kernel lock automatically; the +lock file remains in place and must not be deleted to recover a session. The +private broker descriptor also remains after shutdown so a retiring broker cannot +remove a replacement broker's descriptor. `attach` checks the recorded endpoint +and replaces a stale descriptor when it starts the next broker. + +The viewer polls screenshots; this is not a video/audio streaming transport. +Optional recordings require an installed FFmpeg with `libx264`; health reports +whether recording is available. Capture can include private desktop contents, so +share the desktop only with participants authorized to see the target account. diff --git a/docs/ZUSE_INTEGRATION.md b/docs/ZUSE_INTEGRATION.md index c4a5c4c..6fe5a37 100644 --- a/docs/ZUSE_INTEGRATION.md +++ b/docs/ZUSE_INTEGRATION.md @@ -22,10 +22,12 @@ and recordings. The agent keeps its own model and subscription. 5. On restore or fork, call `rebind` with a new generation before allowing access. On workspace shutdown, call `shutdown`; the provider owns the desktop itself. -For an agent, grant `observe`, `input-control`, and optionally `viewer-read` for -state inspection. Recording scopes are separate. For a person watching, grant -`viewer-read`; add `input-control` only when they may take over. Keep host/admin -credentials in Zuse's backend. Grant tokens expire and can be revoked early. +For an agent, grant `observe`, `input-control`, `viewer-read`, and `presence` +for the multiplayer MCP workflow. Recording scopes are separate. For a person +watching with a named cursor, grant `viewer-read` and `presence`; add +`input-control` only when they may take over. The `share --role agent`, +`share --role viewer`, and `share --role controller` commands create these scope +sets. Keep host/admin credentials in Zuse's backend. Grant tokens expire and can be revoked early. The agent loop is **acquire → observe → input → observe**. Renew its lease while working. A person taking control interrupts agent input; after control is returned, @@ -35,7 +37,8 @@ click or text action is never automatically repeated. The complete commands, credential format, HTTP methods, errors and limitations are in [the service contract](DESKTOP_SERVICE.md). A host proxy example is in [desktop-proxy.mjs](../examples/desktop-proxy.mjs). Use a stable scoped grant per -viewer session; Zuse still validates its own login on every proxy request. +viewer session; configure the broker with the exact external viewer origin +and any proxy base path. Zuse still validates its own login on every proxy request. ## What still needs a Zuse test diff --git a/native/macos/Driver.swift b/native/macos/Driver.swift index 6cfcc15..cae823e 100644 --- a/native/macos/Driver.swift +++ b/native/macos/Driver.swift @@ -1,4 +1,5 @@ import Foundation +import Darwin import AppKit import ApplicationServices import ScreenCaptureKit @@ -22,6 +23,41 @@ struct SavedElement { var visualCapturedAt: Date? var lastCaptureImage: CGImage? var virtualCursor: CGPoint? + var desktopHeldModifiers: CGEventFlags = [] + // Each persistent helper connection owns its own Driver and file descriptor. + // Keep the inode stable: kernel flock releases on disconnect/crash, without + // PID reuse checks or racing stale socket/file deletion. + private var desktopBrokerLock: Int32 = -1 + deinit { + if desktopBrokerLock >= 0 { Darwin.close(desktopBrokerLock) } + } + func acquireDesktopBrokerLock() throws -> [String: Any] { + if desktopBrokerLock >= 0 { return ["locked": true] } + let directory = "/tmp/opcode-cu-\(getuid())" + do { + try FileManager.default.createDirectory(atPath: directory, withIntermediateDirectories: false, attributes: [.posixPermissions: 0o700]) + } catch { /* Validate an existing private directory below. */ } + var directoryInfo = stat() + guard lstat(directory, &directoryInfo) == 0, directoryInfo.st_uid == getuid(), + (directoryInfo.st_mode & S_IFMT) == S_IFDIR, (directoryInfo.st_mode & 0o077) == 0 else { + throw DriverFailure(code: "permission_denied", message: "Unsafe desktop lock directory.") + } + let fd = Darwin.open(directory + "/desktop-broker.lock", O_CREAT | O_RDWR | O_NOFOLLOW | O_CLOEXEC, 0o600) + guard fd >= 0 else { throw DriverFailure(code: "permission_denied", message: "Cannot open desktop broker lock.") } + var info = stat() + guard fstat(fd, &info) == 0, info.st_uid == getuid(), (info.st_mode & S_IFMT) == S_IFREG, + (info.st_mode & 0o077) == 0, info.st_nlink == 1 else { + Darwin.close(fd) + throw DriverFailure(code: "permission_denied", message: "Unsafe desktop broker lock file.") + } + guard flock(fd, LOCK_EX | LOCK_NB) == 0 else { + Darwin.close(fd) + throw DriverFailure(code: "lease_conflict", message: "A broker already owns this display. Reuse its endpoint.") + } + desktopBrokerLock = fd + return ["locked": true] + } + let desktopKeyCodes: [String: CGKeyCode] = ["a":0,"s":1,"d":2,"f":3,"h":4,"g":5,"z":6,"x":7,"c":8,"v":9,"b":11,"q":12,"w":13,"e":14,"r":15,"y":16,"t":17,"1":18,"2":19,"3":20,"4":21,"6":22,"5":23,"9":25,"7":26,"8":28,"0":29,"o":31,"u":32,"i":34,"p":35,"l":37,"j":38,"k":40,"n":45,"m":46,"Enter":36,"Tab":48,"Space":49,"Backspace":51,"Escape":53,"Meta":55,"Shift":56,"Alt":58,"Control":59,"Home":115,"PageUp":116,"Delete":117,"End":119,"PageDown":121,"ArrowLeft":123,"ArrowRight":124,"ArrowDown":125,"ArrowUp":126,"F1":122,"F2":120,"F3":99,"F4":118,"F5":96,"F6":97,"F7":98,"F8":100,"F9":101,"F10":109,"F11":103,"F12":111] let foregroundAllowed = ProcessInfo.processInfo.environment["JEV_INTERACTION_MODE"] == "foreground" var activatedRenderers = Set() var observationErrors = 0 @@ -592,8 +628,230 @@ struct SavedElement { return found.values.sorted { $0.name < $1.name } } + // Remote desktop operations stay in this permission-owning helper. The broker + // serializes physical input; participant cursors are overlays, not HID devices. + func desktopWindows() throws -> [[String: Any]] { + guard CGPreflightScreenCaptureAccess() else { throw DriverFailure(code: "permission_denied", message: "Grant Screen Recording to Opcode.") } + let entries = CGWindowListCopyWindowInfo([.optionOnScreenOnly, .excludeDesktopElements], kCGNullWindowID) as? [[String: Any]] ?? [] + return entries.compactMap { item in + guard let id = item[kCGWindowNumber as String] as? Int, + let pid = item[kCGWindowOwnerPID as String] as? Int, pid > 0, + let bounds = item[kCGWindowBounds as String] as? NSDictionary, + let rect = CGRect(dictionaryRepresentation: bounds), rect.width > 0, rect.height > 0, + (item[kCGWindowLayer as String] as? Int) == 0 else { return nil } + let frame = rect.integral + return ["id": id, "pid": pid, "title": item[kCGWindowName as String] as? String ?? "", + "x": Double(frame.minX), "y": Double(frame.minY), "width": Int(frame.width), "height": Int(frame.height)] + } + } + + func desktopGeometry(_ request: [String: Any]) throws -> [String: Any] { + guard let target = request["target"] as? [String: Any], let kind = target["kind"] as? String else { + throw DriverFailure(code: "InvalidRequest", message: "Desktop target required.") + } + if kind == "display" { + let id = CGMainDisplayID(), frame = CGDisplayBounds(id).integral + return ["id": Int(id), "x": Double(frame.minX), "y": Double(frame.minY), "width": Int(frame.width), "height": Int(frame.height)] + } + guard kind == "window", let id = target["id"] as? Int, + let window = try desktopWindows().first(where: { $0["id"] as? Int == id }) else { + throw DriverFailure(code: "stale_observation", message: "Window is no longer visible.") + } + return window + } + + func desktopState() -> [String: Any] { + let cursor = CGEvent(source: nil)?.location ?? .zero + var focus = 0 + if let app = NSWorkspace.shared.frontmostApplication { + let ax = AXUIElementCreateApplication(app.processIdentifier) + if let raw = value(ax, "AXFocusedWindow"), CFGetTypeID(raw) == AXUIElementGetTypeID() { + focus = Int(backgroundInput.windowID(raw as! AXUIElement) ?? 0) + } + } + return ["focus": focus, "x": cursor.x, "y": cursor.y] + } + + func desktopCapture(_ request: [String: Any]) async throws -> [String: Any] { + guard CGPreflightScreenCaptureAccess() else { throw DriverFailure(code: "permission_denied", message: "Grant Screen Recording to Opcode.") } + guard #available(macOS 14.0, *) else { throw DriverFailure(code: "unsupported", message: "Remote desktop requires macOS 14 or later.") } + let geometry = try desktopGeometry(request) + let width = geometry["width"] as! Int, height = geometry["height"] as! Int + guard width > 0, height > 0, width <= 8192, height <= 8192 else { + throw DriverFailure(code: "unsupported", message: "Desktop dimensions exceed the capture limit.") + } + let content = try await SCShareableContent.excludingDesktopWindows(false, onScreenWindowsOnly: true) + let filter: SCContentFilter + if (request["target"] as? [String: Any])?["kind"] as? String == "window" { + guard let window = content.windows.first(where: { Int($0.windowID) == geometry["id"] as? Int }) else { + throw DriverFailure(code: "stale_observation", message: "Window disappeared before capture.") + } + filter = SCContentFilter(desktopIndependentWindow: window) + } else { + guard let display = content.displays.first(where: { $0.displayID == CGMainDisplayID() }) else { + throw DriverFailure(code: "display_unavailable", message: "Primary display is unavailable.") + } + filter = SCContentFilter(display: display, excludingWindows: []) + } + let config = SCStreamConfiguration() + config.width = width; config.height = height; config.showsCursor = false + config.ignoreShadowsSingleWindow = true; config.ignoreGlobalClipSingleWindow = true + let image = try await SCScreenshotManager.captureImage(contentFilter: filter, configuration: config) + guard NSDictionary(dictionary: geometry).isEqual(to: try desktopGeometry(request)), + let png = NSBitmapImageRep(cgImage: image).representation(using: .png, properties: [:]), png.count <= 12_000_000 else { + throw DriverFailure(code: "stale_observation", message: "Display changed during capture or image exceeds limit.") + } + var sample = [UInt8](repeating: 0, count: 64 * 48 * 4) + let sampled = sample.withUnsafeMutableBytes { bytes -> Bool in + guard let context = CGContext(data: bytes.baseAddress, width: 64, height: 48, bitsPerComponent: 8, bytesPerRow: 64 * 4, + space: CGColorSpaceCreateDeviceRGB(), bitmapInfo: CGImageAlphaInfo.premultipliedLast.rawValue) else { return false } + context.draw(image, in: CGRect(x: 0, y: 0, width: 64, height: 48)); return true + } + guard sampled else { throw DriverFailure(code: "CaptureFailed", message: "Could not sample desktop frame.") } + return ["geometry": geometry, "base64": png.base64EncodedString(), "sample": Data(sample).base64EncodedString()] + } + + func desktopInput(_ request: [String: Any]) throws -> [String: Any] { + try requireAX() + guard request["foregroundApproved"] as? Bool == true, + let action = request["action"] as? [String: Any], let kind = action["kind"] as? String else { + throw DriverFailure(code: "permission_denied", message: "Desktop input requires explicit foreground approval.") + } + let releasing = kind == "keyUp" || kind == "buttonUp" + let geometry = try desktopGeometry(request) + if !releasing { + guard let expected = request["expectedGeometry"] as? [String: Any], NSDictionary(dictionary: expected).isEqual(to: geometry) else { + throw DriverFailure(code: "stale_observation", message: "Desktop geometry changed before input.") + } + if (request["target"] as? [String: Any])?["kind"] as? String == "window", kind != "focus" { + guard desktopState()["focus"] as? Int == geometry["id"] as? Int else { + throw DriverFailure(code: "stale_observation", message: "Focus the target window and observe again before physical input.") + } + } + if let app = NSWorkspace.shared.frontmostApplication { + let ax = AXUIElementCreateApplication(app.processIdentifier) + if let raw = value(ax, "AXFocusedUIElement"), CFGetTypeID(raw) == AXUIElementGetTypeID() { + let field = raw as! AXUIElement + guard string(field, "AXRole") != "AXSecureTextField", string(field, "AXSubrole") != "AXSecureTextField" else { + throw DriverFailure(code: "permission_denied", message: "Remote input is disabled in protected fields.") + } + } + } + } + guard let originX = geometry["x"] as? Double, let originY = geometry["y"] as? Double else { + throw DriverFailure(code: "stale_observation", message: "Desktop coordinate origin is unavailable.") + } + let origin = CGPoint(x: originX, y: originY) + let width = geometry["width"] as! Int, height = geometry["height"] as! Int + if (request["target"] as? [String: Any])?["kind"] as? String == "window", ["scroll", "buttonDown"].contains(kind) { + let bounds = CGRect(x: origin.x, y: origin.y, width: Double(width), height: Double(height)) + guard let cursor = CGEvent(source: nil)?.location, bounds.contains(cursor) else { + throw DriverFailure(code: "stale_observation", message: "Hover inside the target window and observe again before scrolling or holding a button.") + } + } + func point(_ x: String, _ y: String) throws -> CGPoint { + guard let px = action[x] as? Double, let py = action[y] as? Double, + px.isFinite, py.isFinite, px >= 0, py >= 0, px < Double(width), py < Double(height) else { + throw DriverFailure(code: "invalid_coordinates", message: "Input must be inside the observed image.") + } + return CGPoint(x: origin.x + px, y: origin.y + py) + } + func mouse(_ type: CGEventType, _ p: CGPoint, _ button: CGMouseButton = .left, count: Int64 = 1) throws { + guard let event = CGEvent(mouseEventSource: nil, mouseType: type, mouseCursorPosition: p, mouseButton: button) else { + throw DriverFailure(code: "unsupported", message: "Could not allocate mouse event.", delivery: "unknown") + } + event.flags = desktopHeldModifiers + event.setIntegerValueField(.mouseEventClickState, value: count); event.post(tap: .cghidEventTap) + } + func code(_ key: String) -> CGKeyCode? { desktopKeyCodes[key] ?? (key.count == 1 ? desktopKeyCodes[key.lowercased()] : nil) } + func keyboard(_ key: String, _ down: Bool, _ flags: CGEventFlags = []) throws { + guard let c = code(key), let event = CGEvent(keyboardEventSource: nil, virtualKey: c, keyDown: down) else { + throw DriverFailure(code: "unsupported", message: "Unsupported named key.") + } + event.flags = flags.union(desktopHeldModifiers); event.post(tap: .cghidEventTap) + } + switch kind { + case "click", "doubleClick", "rightClick", "hover", "drag": + let start = try point("x", "y") + let end = kind == "drag" ? try point("toX", "toY") : start + if kind == "hover" { try mouse(.mouseMoved, start) } + else if kind == "rightClick" { try mouse(.rightMouseDown, start, .right); try mouse(.rightMouseUp, start, .right) } + else { + let count = kind == "doubleClick" ? 2 : 1 + for index in 1...count { + try mouse(.leftMouseDown, start, count: Int64(index)) + if kind == "drag" { + for step in 1...12 { + let f = Double(step) / 12 + try mouse(.leftMouseDragged, CGPoint(x: start.x + (end.x - start.x) * f, y: start.y + (end.y - start.y) * f)) + Thread.sleep(forTimeInterval: 0.01) + } + } + try mouse(.leftMouseUp, end, count: Int64(index)) + } + } + case "buttonDown", "buttonUp": + guard let button = action["button"] as? Int, (1...3).contains(button) else { throw DriverFailure(code: "InvalidRequest", message: "Invalid button.") } + let p = CGEvent(source: nil)?.location ?? .zero + let down = kind == "buttonDown" + try mouse(button == 1 ? (down ? .leftMouseDown : .leftMouseUp) : button == 3 ? (down ? .rightMouseDown : .rightMouseUp) : (down ? .otherMouseDown : .otherMouseUp), p, button == 1 ? .left : button == 3 ? .right : .center) + case "key", "keyDown", "keyUp": + guard let key = action["key"] as? String else { throw DriverFailure(code: "InvalidRequest", message: "Missing key.") } + let parts = key.components(separatedBy: "+") + guard !parts.isEmpty, parts.allSatisfy({ code($0) != nil }), kind == "key" || parts.count == 1 else { + throw DriverFailure(code: "unsupported", message: "Unsupported key combination.") + } + let modifiers: [String: CGEventFlags] = ["Meta": .maskCommand, "Control": .maskControl, "Alt": .maskAlternate, "Shift": .maskShift] + if kind == "key" { + guard parts.dropLast().allSatisfy({ modifiers[$0] != nil }) else { throw DriverFailure(code: "unsupported", message: "Only modifiers may precede a key.") } + let flags = parts.dropLast().reduce(CGEventFlags()) { $0.union(modifiers[$1] ?? []) } + try keyboard(parts.last!, true, flags); try keyboard(parts.last!, false, flags) + } else { + if let modifier = modifiers[key] { + if kind == "keyDown" { desktopHeldModifiers.insert(modifier) } else { desktopHeldModifiers.remove(modifier) } + } + try keyboard(key, kind == "keyDown") + } + case "text": + guard let text = action["text"] as? String, !text.isEmpty, text.utf16.count <= 16000, !text.contains("\0") else { throw DriverFailure(code: "InvalidRequest", message: "Invalid text.") } + // Quartz limits Unicode payloads; chunk on Unicode scalar boundaries. + var chunks: [[UInt16]] = [[]] + for scalar in text.unicodeScalars { + let units = Array(String(scalar).utf16) + if chunks[chunks.count - 1].count + units.count > 20 { chunks.append([]) } + chunks[chunks.count - 1].append(contentsOf: units) + } + for units in chunks { + guard let down = CGEvent(keyboardEventSource: nil, virtualKey: 0, keyDown: true), let up = CGEvent(keyboardEventSource: nil, virtualKey: 0, keyDown: false) else { throw DriverFailure(code: "unsupported", message: "Could not allocate text event.", delivery: "unknown") } + units.withUnsafeBufferPointer { buffer in + down.keyboardSetUnicodeString(stringLength: units.count, unicodeString: buffer.baseAddress!); up.keyboardSetUnicodeString(stringLength: units.count, unicodeString: buffer.baseAddress!) + } + down.post(tap: .cghidEventTap); up.post(tap: .cghidEventTap) + } + case "scroll": + guard let amount = action["amount"] as? Int, (-30...30).contains(amount), let axis = action["axis"] as? String, ["x", "y"].contains(axis), + let event = CGEvent(scrollWheelEvent2Source: nil, units: .line, wheelCount: 2, wheel1: axis == "y" ? Int32(-amount) : 0, wheel2: axis == "x" ? Int32(-amount) : 0, wheel3: 0) else { throw DriverFailure(code: "InvalidRequest", message: "Invalid scroll.") } + event.post(tap: .cghidEventTap) + case "focus": + guard let id = action["windowId"] as? Int, let item = try desktopWindows().first(where: { $0["id"] as? Int == id }), let pid = item["pid"] as? Int else { throw DriverFailure(code: "stale_observation", message: "Window disappeared.") } + let ax = AXUIElementCreateApplication(pid_t(pid)) + guard let windows = value(ax, "AXWindows") as? [AXUIElement], let window = windows.first(where: { backgroundInput.windowID($0) == CGWindowID(id) }) else { throw DriverFailure(code: "unsupported", message: "Window is not accessible.") } + NSRunningApplication(processIdentifier: pid_t(pid))?.activate(options: []) + guard AXUIElementPerformAction(window, "AXRaise" as CFString) == .success else { throw DriverFailure(code: "ActionOutcomeUnknown", message: "Window focus unconfirmed.", delivery: "unknown") } + default: throw DriverFailure(code: "unsupported", message: "Unsupported desktop action.") + } + return ["delivery": "dispatchedUnverified"] + } + func handle(_ request: [String: Any]) async throws -> Any { switch request["method"] as? String { + case "desktopKeys": return desktopKeyCodes.keys.sorted() + case "desktopLock": return try acquireDesktopBrokerLock() + case "desktopGeometry": return try desktopGeometry(request) + case "desktopWindows": return try desktopWindows() + case "desktopState": return desktopState() + case "desktopCapture": return try await desktopCapture(request) + case "desktopInput": return try desktopInput(request) case "installedApps": return installedApps().map { ["id": $0.id, "name": $0.name] } case "launchApp": guard let id = request["appId"] as? String, let target = installedApps().first(where: { $0.id == id }) else { diff --git a/native/macos/Helper.swift b/native/macos/Helper.swift index de9e9f0..44ec9ce 100644 --- a/native/macos/Helper.swift +++ b/native/macos/Helper.swift @@ -8,6 +8,40 @@ import ApplicationServices @main struct HelperMain { @MainActor static var busy = false @MainActor static var guide: Process? + @MainActor static var activeClients = 0 + // Blocking sockets must not occupy Swift's cooperative executor: enough + // idle peers would otherwise prevent every pending reply from resuming. + // Admission caps this queue at 32 client operations plus one accept. + static let socketIO = DispatchQueue(label: "com.opcodehq.helper.socket-io", qos: .userInitiated, attributes: .concurrent) + static func blockingIO(_ operation: @escaping () -> T) async -> T { + await withCheckedContinuation { continuation in + socketIO.async { continuation.resume(returning: operation()) } + } + } + static func readChunk(_ client: Int32) async -> Data? { + await blockingIO { + var bytes = [UInt8](repeating: 0, count: 8192) + while true { + let count = Darwin.read(client, &bytes, bytes.count) + if count < 0 && errno == EINTR { continue } + return count > 0 ? Data(bytes.prefix(count)) : nil + } + } + } + static func writeReply(_ data: Data, client: Int32) async -> Bool { + await blockingIO { + data.withUnsafeBytes { raw -> Bool in + var offset = 0 + while offset < raw.count { + let count = Darwin.write(client, raw.baseAddress!.advanced(by: offset), raw.count - offset) + if count < 0 && errno == EINTR { continue } + if count <= 0 { return false } + offset += count + } + return true + } + } + } @MainActor static func showPermissionGuide(_ kind: String) { if guide?.isRunning == true { guide?.terminate() } let process = Process() @@ -90,32 +124,28 @@ import ApplicationServices } guard bound == 0, chmod(path, 0o600) == 0, listen(server, 16) == 0 else { exit(1) } while true { - let client = await Task.detached { accept(server, nil, nil) }.value + let client = await blockingIO { Darwin.accept(server, nil, nil) } guard client >= 0 else { continue } var uid: uid_t = 0, gid: gid_t = 0 guard getpeereid(client, &uid, &gid) == 0, uid == getuid() else { Darwin.close(client); continue } + guard activeClients < 32 else { Darwin.close(client); continue } + // A peer that stops reading must not retain a reply thread forever. + var writeTimeout = timeval(tv_sec: 10, tv_usec: 0) + guard setsockopt(client, SOL_SOCKET, SO_SNDTIMEO, &writeTimeout, socklen_t(MemoryLayout.size)) == 0 else { + Darwin.close(client); continue + } + activeClients += 1 let driver = Driver() - Task.detached { - defer { Darwin.close(client) } + Task { @MainActor in + defer { Darwin.close(client); activeClients -= 1 } var buffer = Data() - var chunk = [UInt8](repeating: 0, count: 8192) - while true { - let count = Darwin.read(client, &chunk, chunk.count) - if count <= 0 { break } - buffer.append(contentsOf: chunk.prefix(count)) + while let chunk = await readChunk(client) { + buffer.append(chunk) while let newline = buffer.firstIndex(of: 10) { let line = Data(buffer[.. 128_000 { return } let data = await reply(line, driver: driver) - let sent = data.withUnsafeBytes { raw -> Bool in - var offset = 0 - while offset < raw.count { - let count = Darwin.write(client, raw.baseAddress!.advanced(by: offset), raw.count - offset) - if count <= 0 { return false } - offset += count - } - return true - } + let sent = await writeReply(data, client: client) if !sent { return } } if buffer.count > 128_000 { break } diff --git a/scripts/test-macos-desktop-live.ts b/scripts/test-macos-desktop-live.ts new file mode 100644 index 0000000..419bdd4 --- /dev/null +++ b/scripts/test-macos-desktop-live.ts @@ -0,0 +1,353 @@ +import assert from "node:assert/strict"; +import { + type ChildProcessWithoutNullStreams, + spawn, + spawnSync, +} from "node:child_process"; +import { randomUUID } from "node:crypto"; +import { mkdtemp, readFile, realpath, rm } from "node:fs/promises"; +import { join, relative, resolve } from "node:path"; +import { createInterface } from "node:readline"; +import { setTimeout as delay } from "node:timers/promises"; +import type { Grant } from "../src/desktop/authority.js"; +import { DesktopController } from "../src/desktop/controller.js"; +import { MacBackend, type MacDriver } from "../src/desktop/mac-backend.js"; +import { + type Action, + type Observation, + scopes, +} from "../src/desktop/protocol.js"; + +if ( + process.platform !== "darwin" || + process.env.CI !== "true" || + process.env.GITHUB_ACTIONS !== "true" +) + throw Error( + "Live Mac desktop tests run only on disposable GitHub macOS CI runners.", + ); +if (!process.env.RUNNER_TEMP || !process.env.CU_NATIVE_DIR) + throw Error( + "Set RUNNER_TEMP and CU_NATIVE_DIR to the CI temporary build directory.", + ); +const runner = await realpath(process.env.RUNNER_TEMP); +const native = await realpath(process.env.CU_NATIVE_DIR); +const withinRunner = relative(runner, native); +if ( + !withinRunner || + withinRunner.startsWith("..") || + withinRunner.startsWith("/") +) + throw Error( + "CU_NATIVE_DIR must be a temporary build directory beneath RUNNER_TEMP.", + ); +const directory = await mkdtemp(join(runner, "opcode-live-")); +const driverProcess = spawn(join(native, "desktop-driver"), [], { + stdio: ["pipe", "pipe", "pipe"], + env: { ...process.env, JEV_INTERACTION_MODE: "background" }, +}); +let fixtureProcess: ChildProcessWithoutNullStreams | undefined; +let controller: DesktopController | undefined; +let heartbeat: ReturnType | undefined; +let diagnostics = ""; +driverProcess.stderr.on("data", (chunk) => { + diagnostics = (diagnostics + chunk).slice(-16_000); +}); +const pending = new Map< + string, + { + resolve(value: unknown): void; + reject(error: Error): void; + timer: ReturnType; + method: string; + } +>(); +function disconnected() { + for (const task of pending.values()) { + clearTimeout(task.timer); + task.reject(Error("Temporary native driver disconnected. " + diagnostics)); + } + pending.clear(); +} +driverProcess.on("error", disconnected); +driverProcess.on("exit", disconnected); +const lines = createInterface({ input: driverProcess.stdout }); +lines.on("line", (line) => { + try { + const reply = JSON.parse(line); + const task = pending.get(reply.id); + if (!task) throw Error("Unexpected native reply ID."); + pending.delete(reply.id); + clearTimeout(task.timer); + if (reply.ok) task.resolve(reply.data); + else { + console.error( + "NATIVE FAILURE", + JSON.stringify({ method: task.method, ...reply.error }), + ); + task.reject(Object.assign(Error(reply.error.message), reply.error)); + } + } catch (error) { + disconnected(); + driverProcess.kill(); + } +}); +const driver: MacDriver = { + async request(method, args = {}, signal) { + signal?.throwIfAborted(); + const id = randomUUID(); + const result = await new Promise((resolve, reject) => { + const timer = setTimeout(() => { + pending.delete(id); + reject(Error(`Native ${method} timed out.`)); + driverProcess.kill(); + }, 15_000); + pending.set(id, { resolve, reject, timer, method }); + driverProcess.stdin.write(JSON.stringify({ ...args, id, method }) + "\n"); + }); + signal?.throwIfAborted(); + return result; + }, +}; +async function poll( + read: () => Promise, + message: string, +): Promise { + for (let attempt = 0; attempt < 50; attempt++) { + const value = await read(); + if (value !== undefined) return value; + await delay(100); + } + throw Error(message); +} +async function stop(child: ChildProcessWithoutNullStreams | undefined) { + if (!child || child.exitCode !== null || child.signalCode !== null) return; + await new Promise((resolve) => { + const force = setTimeout(() => child.kill("SIGKILL"), 1000); + child.once("exit", () => { + clearTimeout(force); + resolve(); + }); + child.kill("SIGTERM"); + }); +} +try { + const status = (await driver.request("status")) as { + accessibility: boolean; + screenRecording: boolean; + }; + assert.equal( + status.accessibility, + true, + "Live CI requires Accessibility; native delivery is unverified without it.", + ); + assert.equal( + status.screenRecording, + true, + "Live CI requires Screen Recording; native capture is unverified without it.", + ); + const fixtureBinary = join(directory, "fixture"); + const compile = spawnSync( + "xcrun", + [ + "swiftc", + "-swift-version", + "5", + "-parse-as-library", + "-framework", + "AppKit", + resolve("tests/fixtures/mac-desktop.swift"), + "-o", + fixtureBinary, + ], + { encoding: "utf8", timeout: 120_000 }, + ); + assert.equal(compile.status, 0, compile.stderr || compile.error?.message); + const title = "Opcode CI " + randomUUID(); + const output = join(directory, "result.json"); + fixtureProcess = spawn(fixtureBinary, [output, title], { + stdio: ["pipe", "pipe", "pipe"], + }); + fixtureProcess.on("error", () => {}); + fixtureProcess.stderr.on("data", (chunk) => { + diagnostics = (diagnostics + chunk).slice(-16_000); + }); + const manifest = await poll(async () => { + try { + return JSON.parse(await readFile(output + ".ready", "utf8")) as { + pid: number; + field: { x: number; y: number }; + save: { x: number; y: number }; + }; + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + return undefined; + } + }, "Fixture did not become ready. " + diagnostics); + const backend = new MacBackend(driver); + const window = await poll( + async () => + (await backend.windows()).find( + (window) => window.title === title && window.pid === manifest.pid, + ), + "Exact fixture window was not found.", + ); + controller = new DesktopController( + backend, + "live-ci", + join(directory, "recordings"), + ); + await controller.init(); + const c = controller; + const call = ( + grant: Grant, + method: string, + args: Record = {}, + ) => c.call({ id: randomUUID(), generation: "live-ci", method, args }, grant); + const agent = c.authority.issue("agent", [...scopes], 60_000); + const human = c.authority.issue("human", [...scopes], 60_000); + const a = (await call(agent, "presence.join", { + name: "CI agent", + role: "agent", + })) as { id: string }; + const h = (await call(human, "presence.join", { + name: "CI human", + role: "human", + })) as { id: string }; + heartbeat = setInterval(() => { + void call(agent, "presence.update", { participantId: a.id }).catch( + () => {}, + ); + void call(human, "presence.update", { participantId: h.id }).catch( + () => {}, + ); + }, 3000); + const before = await backend.state(); + await call(agent, "presence.update", { + participantId: a.id, + cursor: { x: 0.2, y: 0.3 }, + }); + await call(human, "presence.update", { + participantId: h.id, + cursor: { x: 0.7, y: 0.8 }, + }); + const after = await backend.state(); + assert.deepEqual( + { x: after.x, y: after.y }, + { x: before.x, y: before.y }, + "Overlay cursors must not move the real Mac pointer.", + ); + const presence = (await call(human, "presence.list")) as { + participants: { cursor: unknown }[]; + }; + assert.equal(presence.participants.length, 2); + assert.deepEqual( + presence.participants.map((participant) => participant.cursor), + [ + { x: 0.2, y: 0.3 }, + { x: 0.7, y: 0.8 }, + ], + ); + const firstLease = (await call(agent, "acquire", { + participantId: a.id, + })) as { id: string }; + await assert.rejects( + call(human, "acquire", { participantId: h.id }), + /controller/, + ); + const old = await c.observe(agent, { kind: "display" }); + const lease = (await call(human, "takeover", { participantId: h.id })) as { + id: string; + }; + await assert.rejects( + call(agent, "input", { + leaseId: firstLease.id, + observationId: old.observationId, + action: { kind: "hover", x: 1, y: 1 }, + }), + /lease|control/i, + ); + console.log( + "PASS capture, overlay cursors, and lease takeover; next: native fixture focus.", + ); + // Focus only our exact disposable window before delivering physical input. + const observed = await c.observe(human, { kind: "display" }); + await call(human, "input", { + leaseId: lease.id, + observationId: observed.observationId, + action: { kind: "focus", windowId: window.id }, + }); + await poll( + async () => + (await backend.state()).focus === window.id ? true : undefined, + "Fixture window did not become focused.", + ); + assert.notEqual( + window.x, + 0, + "Fixture must exercise a nonzero horizontal window origin.", + ); + const target = { kind: "window" as const, id: window.id }; + async function act(action: Action | ((observation: Observation) => Action)) { + await delay(200); + const observation = await c.observe(human, target); + const png = Buffer.from(observation.image.base64, "base64"); + assert.equal(png.readUInt32BE(16), observation.image.width); + assert.equal(png.readUInt32BE(20), observation.image.height); + assert.equal(observation.imageToDesktop.scaleX, 1); + assert.equal(observation.imageToDesktop.scaleY, 1); + await call(human, "input", { + leaseId: lease.id, + observationId: observation.observationId, + action: typeof action === "function" ? action(observation) : action, + }); + } + const click = + (point: { x: number; y: number }) => + (observation: Observation): Action => ({ + kind: "click", + x: point.x - observation.desktopBounds.x, + y: point.y - observation.desktopBounds.y, + }); + console.log("PASS fixture focus; next: native field click."); + await act(click(manifest.field)); + const text = "Opcode shared desktop CI Unicode α🙂 and chunked text verified"; + await poll( + async () => + (await backend.state()).focus === window.id ? true : undefined, + "Native field click moved focus away from the fixture; check nonzero window-origin coordinate mapping.", + ); + console.log( + "PASS field click retained exact window focus; next: native Unicode text.", + ); + await act({ kind: "text", text }); + console.log("PASS text dispatch; next: native save click."); + await act(click(manifest.save)); + const result = await poll(async () => { + try { + return JSON.parse(await readFile(output, "utf8")); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + return undefined; + } + }, "Native save click did not produce fixture output."); + assert.deepEqual( + result, + { saved: true, text }, + "Native text/click outcome must match exactly.", + ); + await call(human, "release", { leaseId: lease.id }); + console.log( + "PASS real Mac capture/point geometry, two independent overlay cursors, exclusive control/takeover, native focus/click/Unicode text/save against disposable fixture.", + ); +} finally { + if (heartbeat) clearInterval(heartbeat); + try { + await controller?.close(); + } finally { + lines.close(); + await stop(driverProcess); + await stop(fixtureProcess); + await rm(directory, { recursive: true, force: true }); + } +} diff --git a/scripts/test-macos-desktop.mjs b/scripts/test-macos-desktop.mjs new file mode 100644 index 0000000..08916da --- /dev/null +++ b/scripts/test-macos-desktop.mjs @@ -0,0 +1,160 @@ +import assert from "node:assert/strict"; +import { spawnSync } from "node:child_process"; +import { join, resolve } from "node:path"; + +if (process.platform !== "darwin") + throw new Error("This native smoke test requires macOS."); +if (!process.env.CU_NATIVE_DIR) + throw new Error( + "Set CU_NATIVE_DIR to the temporary compiled native output directory.", + ); +const binary = join(resolve(process.env.CU_NATIVE_DIR), "desktop-driver"); + +// Run only the temporary standalone binary. Never install, launch, stop, or +// reconnect the user's installed permission-owning helper. +function run(requests) { + const child = spawnSync(binary, [], { + input: requests.map((request) => JSON.stringify(request)).join("\n") + "\n", + encoding: "utf8", + timeout: 15_000, + maxBuffer: 1_000_000, + env: { ...process.env, JEV_INTERACTION_MODE: "background" }, + }); + if (child.error) throw child.error; + assert.equal( + child.signal, + null, + "Native driver unexpectedly terminated by a signal.", + ); + assert.equal( + child.status, + 0, + `Native driver exited unsuccessfully: ${child.stderr}`, + ); + const replies = child.stdout + .trim() + .split("\n") + .map((line) => JSON.parse(line)); + assert.equal( + replies.length, + requests.length, + "Expected exactly one JSONL response per request.", + ); + for (let index = 0; index < replies.length; index++) { + assert.equal( + replies[index].id, + requests[index].id, + "Native response ID mismatch.", + ); + assert.equal(typeof replies[index].ok, "boolean"); + } + return replies; +} +function denied(reply, codes) { + assert.equal(reply.ok, false, `${reply.id} must fail closed.`); + assert.ok( + codes.includes(reply.error?.code), + `${reply.id}: unexpected error code ${reply.error?.code}`, + ); + assert.equal(typeof reply.error.message, "string"); + assert.equal(reply.error.delivery, "notDispatched"); + assert.equal(reply.data, undefined); +} +const [statusReply, geometryReply, stateReply, keysReply] = run([ + { id: "status", method: "status" }, + { id: "geometry", method: "desktopGeometry", target: { kind: "display" } }, + { id: "state", method: "desktopState" }, + { id: "keys", method: "desktopKeys" }, +]); +assert.equal(statusReply.ok, true); +const status = statusReply.data; +assert.equal(status.platform, "macOS"); +assert.equal(typeof status.accessibility, "boolean"); +assert.equal(typeof status.screenRecording, "boolean"); +assert.equal(geometryReply.ok, true); +const geometry = geometryReply.data; +for (const key of ["id", "width", "height"]) { + assert.ok( + Number.isInteger(geometry[key]) && geometry[key] > 0, + `Invalid desktop geometry ${key}.`, + ); +} +for (const key of ["x", "y"]) + assert.ok(Number.isFinite(geometry[key]), `Invalid desktop geometry ${key}.`); +assert.equal(stateReply.ok, true); +assert.ok( + Number.isInteger(stateReply.data.focus) && stateReply.data.focus >= 0, +); +for (const key of ["x", "y"]) assert.ok(Number.isFinite(stateReply.data[key])); + +assert.equal(keysReply.ok, true); +assert.ok(Array.isArray(keysReply.data)); +assert.ok( + keysReply.data.every((key) => typeof key === "string" && key.length > 0), +); +assert.equal(new Set(keysReply.data).size, keysReply.data.length); +for (const key of ["a", "Enter", "Meta", "Control", "Shift", "Alt"]) + assert.ok(keysReply.data.includes(key)); + +const requests = [ + { id: "missing-target", method: "desktopGeometry" }, + { + id: "unknown-target", + method: "desktopGeometry", + target: { kind: "not-a-desktop" }, + }, + { id: "malformed-target", method: "desktopGeometry", target: "display" }, + { + id: "malformed-window", + method: "desktopGeometry", + target: { kind: "window", id: "not-an-id" }, + }, + { + id: "unknown-selector", + method: "desktopDoesNotExist", + target: { kind: "display" }, + }, + // With AX granted, explicit foreground approval still has to be present. + // This request can never produce valid input, even if permissions change. + { + id: "unapproved-input", + method: "desktopInput", + foregroundApproved: false, + target: { kind: "not-a-desktop" }, + action: { kind: "not-an-action" }, + }, +]; +// Capture with an invalid target cannot read private pixels even if permission +// becomes granted between status and this request. Never request valid capture. +if (!status.screenRecording) + requests.push({ + id: "capture-without-permission", + method: "desktopCapture", + target: { kind: "not-a-desktop" }, + }); +if (!status.accessibility) + requests.push({ + id: "input-without-permission", + method: "desktopInput", + foregroundApproved: false, + target: { kind: "not-a-desktop" }, + action: { kind: "not-an-action" }, + }); +const replies = run(requests); +for (const reply of replies) { + if (reply.id === "capture-without-permission") + denied(reply, ["permission_denied"]); + else if (reply.id === "input-without-permission") + denied(reply, ["AccessibilityDenied"]); + else if (reply.id === "unapproved-input") + denied(reply, ["AccessibilityDenied", "permission_denied"]); + else if (reply.id === "unknown-target" || reply.id === "malformed-window") + denied(reply, ["stale_observation"]); + else denied(reply, ["InvalidRequest"]); +} +console.log( + `PASS native Mac desktop selectors, geometry/state shapes, malformed targets, and permission/input approval gates (${4 + requests.length} requests; no screenshots or input dispatched).`, +); +console.log( + `Permission-dependent denial checks: Screen Recording ${status.screenRecording ? "skipped (already granted)" : "verified"}, Accessibility ${status.accessibility ? "skipped (already granted)" : "verified"}.`, +); diff --git a/src/desktop/backend.ts b/src/desktop/backend.ts index 8f902a6..25b5a03 100644 --- a/src/desktop/backend.ts +++ b/src/desktop/backend.ts @@ -1,10 +1,11 @@ +import { execFile } from "node:child_process"; import { readdir, readFile } from "node:fs/promises"; -import { join } from "node:path"; import { homedir } from "node:os"; -import { execFile } from "node:child_process"; +import { join } from "node:path"; import { promisify } from "node:util"; import { ppmToPNG } from "../linux/capture.js"; -import { failure, type Target, type Action } from "./protocol.js"; +import { type Action, failure, type Target } from "./protocol.js"; + const exec = promisify(execFile); export interface Geometry { id: number; @@ -15,7 +16,25 @@ export interface Geometry { pid?: number; title?: string; } -export class X11Backend { +export interface DesktopBackend { + readonly env: NodeJS.ProcessEnv; + readonly capabilities?: { + backend: string; + resize: string | false; + limitations: string[]; + }; + geometry(target: Target): Promise; + windows(): Promise; + apps(): Promise<{ id: string; name: string; path?: string }[]>; + state(): Promise<{ focus: number; appClass?: string; x: number; y: number }>; + capture( + target: Target, + ): Promise<{ geometry: Geometry; png: Buffer; ppm: Buffer; sample: Buffer }>; + prepareAction(action: Action): Promise; + input(target: Target, action: Action, signal: AbortSignal): Promise; + resize(width: number, height: number): Promise; +} +export class X11Backend implements DesktopBackend { constructor( readonly helper: string, readonly env: NodeJS.ProcessEnv, diff --git a/src/desktop/cli.ts b/src/desktop/cli.ts index 07b2478..5b4cf15 100644 --- a/src/desktop/cli.ts +++ b/src/desktop/cli.ts @@ -1,16 +1,24 @@ -import { parseArgs } from "node:util"; -import { readFile, writeFile, mkdir, lstat, open } from "node:fs/promises"; -import { dirname, join, resolve } from "node:path"; -import { fileURLToPath } from "node:url"; import { spawn } from "node:child_process"; -import { randomUUID } from "node:crypto"; -import { Config, startDesktop, registryPath } from "./server.js"; +import { constants } from "node:fs"; +import { lstat, mkdir, open, readFile } from "node:fs/promises"; +import { dirname, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import { parseArgs } from "node:util"; +import { + desktopCall, + readCredential, + serviceURL, + writeCredential, +} from "./client.js"; import { errorJSON, failure, - methodSchema, methodArguments, + methodSchema, } from "./protocol.js"; +import { openSSHTunnel } from "./remote.js"; +import { Config, registryPath, startDesktop } from "./server.js"; + const { positionals, values } = parseArgs({ allowPositionals: true, options: { @@ -27,30 +35,25 @@ const { positionals, values } = parseArgs({ args: { type: "string" }, id: { type: "string" }, help: { type: "boolean" }, + endpoint: { type: "string" }, + ssh: { type: "string" }, + "remote-credential-file": { type: "string" }, + "local-port": { type: "string" }, + output: { type: "string" }, + subject: { type: "string" }, + role: { type: "string" }, + "ttl-ms": { type: "string" }, }, }); const command = positionals[0]; -const display = values.display ?? process.env.DISPLAY; +const display = + values.display ?? + (process.platform === "darwin" ? "macos" : process.env.DISPLAY); async function descriptor() { if (!display && !values["credential-file"]) throw failure("invalid_request", "Supply --display or --credential-file."); const path = values["credential-file"] ?? registryPath(display!); - const info = await lstat(path); - if ( - info.isSymbolicLink() || - !info.isFile() || - info.uid !== process.getuid?.() || - info.mode & 0o077 - ) - throw failure( - "permission_denied", - "Credential file must be private and owned by this user.", - ); - return JSON.parse(await readFile(path, "utf8")) as { - endpoint: string; - token: string; - generation: string; - }; + return readCredential(path); } async function call( method: string, @@ -58,47 +61,34 @@ async function call( signal?: AbortSignal, ) { const d = await descriptor(); - const endpoint = new URL(d.endpoint); - if (!["127.0.0.1", "localhost", "[::1]"].includes(endpoint.hostname)) - throw failure( - "permission_denied", - "Use a trusted local proxy for remote service access.", - ); - const request = { - id: values.id ?? randomUUID(), - generation: values.generation ?? d.generation, + return desktopCall( + { ...d, generation: values.generation ?? d.generation }, method, args, - }; - const response = await fetch(d.endpoint + "/rpc", { - method: "POST", - headers: { - Authorization: "Bearer " + d.token, - "Content-Type": "application/json", - }, - body: JSON.stringify(request), - signal: signal ?? AbortSignal.timeout(20000), - }); - const result = (await response.json()) as { - ok: boolean; - result: unknown; - error?: { code: string; message: string; delivery: string }; - }; - if (!result.ok) - throw Object.assign(new Error(result.error?.message), result.error); - return result.result; + { id: values.id, signal }, + ); } + try { if (values.help || !command) { - console.log(`Opcode display service (Linux X11, protocol 1) + console.log(`Opcode shared desktop service (macOS / Linux X11, protocol 1) cu desktop-api attach --display :N --generation HOST_GENERATION [--authority PATH] cu desktop-api serve --config PATH cu desktop-api call --display :N --method observe --args '{"target":{"kind":"display"}}' cu desktop-api mcp --display :N +cu desktop-api attach --display macos --generation HOST_GENERATION +cu desktop-api share --display macos --subject alice --role controller --output /private/alice.json [--endpoint https://mac.tailnet.ts.net] +cu desktop-api viewer --credential-file /private/alice.json +cu desktop-api tunnel --ssh user@mac --remote-credential-file /private/alice.json --output /private/local-alice.json [--local-port 4311] +Keep tunnel running; use its output credential for MCP or the viewer. +Roles: viewer (watch and cursor), controller (watch, cursor, input), agent (observe and input). +Remote credentials use HTTPS or an authenticated loopback SSH tunnel. Use --credential-file PATH for a scoped client instead of local host authority. Raw capture does not use models. attach preserves the provider display. serve owns only the broker; exit never destroys an attached display.`); } else if (command === "serve") { + if (values["credential-file"]) + throw failure("invalid_request", "serve cannot use a remote credential."); if (!values.config) throw failure("invalid_request", "Supply --config."); const config = Config.parse( JSON.parse(await readFile(values.config, "utf8")), @@ -115,6 +105,11 @@ serve owns only the broker; exit never destroys an attached display.`); for (const sig of ["SIGTERM", "SIGINT"] as const) process.once(sig, () => void service.close().then(() => process.exit(0))); } else if (command === "attach") { + if (values["credential-file"]) + throw failure( + "invalid_request", + "attach is local; use a scoped credential with call, mcp, or viewer.", + ); if (!display || !values.generation) throw failure( "invalid_request", @@ -147,9 +142,50 @@ serve owns only the broker; exit never destroys an attached display.`); const path = registryPath(display), folder = dirname(path); await mkdir(folder, { recursive: true, mode: 0o700 }); + const info = await lstat(folder); + if ( + !info.isDirectory() || + info.isSymbolicLink() || + info.uid !== process.getuid?.() || + info.mode & 0o077 + ) + throw failure("permission_denied", "Unsafe broker directory."); const configPath = path + ".config"; - await writeFile(configPath, JSON.stringify(config), { mode: 0o600 }); - const log = await open(path + ".log", "a", 0o600); + const configFile = await open( + configPath, + constants.O_WRONLY | constants.O_CREAT | constants.O_NOFOLLOW, + 0o600, + ); + try { + const metadata = await configFile.stat(); + if ( + !metadata.isFile() || + metadata.uid !== process.getuid?.() || + metadata.mode & 0o077 + ) + throw failure("permission_denied", "Unsafe broker config."); + await configFile.truncate(0); + await configFile.writeFile(JSON.stringify(config)); + } finally { + await configFile.close(); + } + const log = await open( + path + ".log", + constants.O_WRONLY | + constants.O_CREAT | + constants.O_APPEND | + constants.O_NOFOLLOW, + 0o600, + ); + const logInfo = await log.stat(); + if ( + !logInfo.isFile() || + logInfo.uid !== process.getuid?.() || + logInfo.mode & 0o077 + ) { + await log.close(); + throw failure("permission_denied", "Unsafe broker log."); + } const child = spawn( process.execPath, [fileURLToPath(import.meta.url), "serve", "--config", configPath], @@ -174,6 +210,112 @@ serve owns only the broker; exit never destroys an attached display.`); "Broker did not become ready. Check the private broker log, display, user, authority and helper path.", ); } + } else if (command === "tunnel") { + if (!values.ssh || !values["remote-credential-file"] || !values.output) + throw failure( + "invalid_request", + "Supply --ssh, --remote-credential-file and --output.", + ); + const tunnel = await openSSHTunnel({ + host: values.ssh, + remoteCredentialFile: values["remote-credential-file"], + output: values.output, + port: Number(values["local-port"] ?? 4311), + }); + console.log( + JSON.stringify({ + ready: true, + endpoint: tunnel.endpoint, + credentialFile: resolve(values.output), + }), + ); + const stop = () => void tunnel.close(); + process.once("SIGTERM", stop); + process.once("SIGINT", stop); + await tunnel.done; + await tunnel.close(); + process.off("SIGTERM", stop); + process.off("SIGINT", stop); + } else if (command === "share") { + if (!values.subject || !values.output) + throw failure( + "invalid_request", + "Supply --subject and --output for a private scoped credential.", + ); + const role = values.role ?? "viewer"; + const roles: Record = { + viewer: ["viewer-read", "presence"], + controller: ["viewer-read", "observe", "presence", "input-control"], + agent: ["observe", "viewer-read", "presence", "input-control"], + }; + if (!Object.hasOwn(roles, role)) + throw failure( + "invalid_request", + "Role must be viewer, controller, or agent.", + ); + const d = await descriptor(); + const endpoint = serviceURL(values.endpoint ?? d.endpoint).href.replace( + /\/$/, + "", + ); + const ttlMs = Number(values["ttl-ms"] ?? 3600000); + if (!Number.isInteger(ttlMs) || ttlMs < 1000 || ttlMs > 3600000) + throw failure( + "invalid_request", + "--ttl-ms must be between 1000 and 3600000.", + ); + const grant = (await call("grant", { + subject: values.subject, + scopes: roles[role], + ttlMs, + })) as { id: string; token: string; generation: string; expiresAt: number }; + try { + await writeCredential(values.output, { + endpoint, + token: grant.token, + generation: grant.generation, + }); + } catch (error) { + await call("revoke", { id: grant.id }).catch(() => {}); + throw error; + } + console.log( + JSON.stringify({ + credentialFile: resolve(values.output), + grantId: grant.id, + role, + expiresAt: grant.expiresAt, + endpoint, + }), + ); + } else if (command === "viewer") { + if (!values["credential-file"]) + throw failure( + "permission_denied", + "Use a scoped --credential-file created by share, never the host descriptor.", + ); + const d = await descriptor(); + const viewerBase = serviceURL(d.endpoint).href.replace(/\/$/, ""); + const sessionResponse = await fetch(viewerBase + "/session", { + headers: { Authorization: "Bearer " + d.token }, + redirect: "error", + signal: AbortSignal.timeout(10000), + }); + const session = (await sessionResponse.json()) as { + admin?: boolean; + scopes?: string[]; + }; + if ( + !sessionResponse.ok || + session.admin || + !session.scopes?.includes("viewer-read") + ) + throw failure( + "permission_denied", + "A scoped viewer credential is required.", + ); + await desktopCall(d, "presence.list", {}); + console.log(viewerBase + "/view#" + encodeURIComponent(d.token)); } else if (command === "call") { if (!values.method) throw failure("invalid_request", "Supply --method."); console.log( @@ -192,6 +334,7 @@ serve owns only the broker; exit never destroys an attached display.`); import("@modelcontextprotocol/sdk/types.js"), ]); const methods = [ + "presence.list", "health", "apps", "windows", @@ -212,6 +355,7 @@ serve owns only the broker; exit never destroys an attached display.`); "recording.delete", ]; const reads = new Set([ + "presence.list", "health", "apps", "windows", @@ -241,21 +385,85 @@ serve owns only the broker; exit never destroys an attached display.`); })), })); let ownedLease: string | undefined; + let participant: { id: string } | undefined; + let presenceEnabled = true; + let presenceBusy = false; + let closing = false; + const observations = new Map< + string, + { target: unknown; image: { width: number; height: number } } + >(); + async function heartbeat() { + if (closing || !presenceEnabled || presenceBusy) return; + presenceBusy = true; + try { + if (!participant) + participant = (await call("presence.join", { role: "agent" })) as { + id: string; + }; + else await call("presence.update", { participantId: participant.id }); + } catch (error) { + if ((error as { code?: string }).code === "permission_denied") + presenceEnabled = false; + participant = undefined; + } finally { + presenceBusy = false; + } + } + await heartbeat(); + const presenceTimer = setInterval(() => void heartbeat(), 4000); + presenceTimer.unref(); server.setRequestHandler(CallToolRequestSchema, async (request, extra) => { const method = methods.find( (m) => "computer_" + m.replaceAll(".", "_") === request.params.name, ); try { if (!method) throw failure("unsupported", "Unknown tool."); - const result = await call( - method, - request.params.arguments ?? {}, - extra.signal, - ); + const args = { ...request.params.arguments }; + if ( + (method === "acquire" || method === "takeover") && + participant && + !args.participantId + ) + args.participantId = participant.id; + if ( + method === "input" && + participant && + typeof args.observationId === "string" + ) { + const observation = observations.get(args.observationId); + const action = args.action as { x?: number; y?: number } | undefined; + if ( + observation && + typeof action?.x === "number" && + typeof action.y === "number" + ) { + await call("presence.update", { + participantId: participant.id, + cursor: { + x: action.x / observation.image.width, + y: action.y / observation.image.height, + target: observation.target, + }, + }).catch(() => {}); + } + } + const result = await call(method, args, extra.signal); if (method === "acquire" || method === "takeover") ownedLease = (result as { id: string }).id; if (method === "release" || method === "stop") ownedLease = undefined; if (method === "observe") { + const observed = result as { + observationId: string; + target: unknown; + image: { width: number; height: number }; + }; + observations.set(observed.observationId, { + target: observed.target, + image: observed.image, + }); + if (observations.size > 32) + observations.delete(observations.keys().next().value!); const r = result as { image: { base64: string; mimeType: string }; [key: string]: unknown; @@ -290,6 +498,12 @@ serve owns only the broker; exit never destroys an attached display.`); } }); server.onclose = () => { + closing = true; + clearInterval(presenceTimer); + if (participant) + void call("presence.leave", { participantId: participant.id }).catch( + () => {}, + ); if (ownedLease) void call("release", { leaseId: ownedLease }).catch(() => {}); }; diff --git a/src/desktop/client.ts b/src/desktop/client.ts new file mode 100644 index 0000000..e7fe29d --- /dev/null +++ b/src/desktop/client.ts @@ -0,0 +1,164 @@ +import { randomUUID } from "node:crypto"; +import { constants } from "node:fs"; +import { mkdir, open, unlink } from "node:fs/promises"; +import { dirname } from "node:path"; +import { z } from "zod"; +import { failure } from "./protocol.js"; + +export const Credential = z.object({ + endpoint: z.string().url(), + token: z.string().min(16).max(4096), + generation: z.string().min(1).max(128), +}); +export type Credential = z.infer; + +export function serviceURL(endpoint: string) { + const url = new URL(endpoint); + const local = ["127.0.0.1", "localhost", "[::1]"].includes(url.hostname); + if ( + (url.protocol !== "https:" && !(local && url.protocol === "http:")) || + url.username || + url.password || + url.search || + url.hash || + !/^(?:\/[a-zA-Z0-9_-]+)*\/?$/.test(url.pathname) + ) + throw failure( + "permission_denied", + "Use HTTPS for remote desktops or HTTP through a loopback SSH tunnel; credentials cannot be embedded in the endpoint.", + ); + return url; +} + +export async function readCredential(path: string): Promise { + const file = await open( + path, + constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK, + ); + try { + const info = await file.stat(); + if ( + !info.isFile() || + info.uid !== process.getuid?.() || + info.mode & 0o077 || + info.size > 16384 + ) + throw failure( + "permission_denied", + "Credential file must be private, bounded, and owned by this user.", + ); + const value = Credential.parse(JSON.parse(await file.readFile("utf8"))); + serviceURL(value.endpoint); + return value; + } finally { + await file.close(); + } +} + +export async function writeCredential(path: string, value: Credential) { + Credential.parse(value); + serviceURL(value.endpoint); + await mkdir(dirname(path), { recursive: true, mode: 0o700 }); + const file = await open( + path, + constants.O_WRONLY | + constants.O_CREAT | + constants.O_EXCL | + constants.O_NOFOLLOW, + 0o600, + ); + try { + await file.writeFile(JSON.stringify(value) + "\n"); + } catch (error) { + await unlink(path).catch(() => {}); + throw error; + } finally { + await file.close(); + } +} + +const reads = new Set([ + "health", + "apps", + "windows", + "state", + "observe", + "recording.list", + "presence.list", +]); +export async function desktopCall( + credential: Credential, + method: string, + args: Record, + options: { id?: string; signal?: AbortSignal; fetch?: typeof fetch } = {}, +): Promise { + const endpoint = serviceURL(credential.endpoint).href.replace(/\/$/, ""); + options.signal?.throwIfAborted(); + let response: Response; + try { + response = await (options.fetch ?? fetch)(endpoint + "/rpc", { + method: "POST", + redirect: "error", + headers: { + Authorization: "Bearer " + credential.token, + "Content-Type": "application/json", + }, + body: JSON.stringify({ + id: options.id ?? randomUUID(), + generation: credential.generation, + method, + args, + }), + signal: options.signal + ? AbortSignal.any([options.signal, AbortSignal.timeout(20000)]) + : AbortSignal.timeout(20000), + }); + } catch { + throw failure( + "connection_lost", + reads.has(method) + ? "Desktop connection failed." + : "Desktop connection failed; the action may have been delivered. Observe before deciding whether to retry.", + reads.has(method) ? "notDispatched" : "unknown", + ); + } + let result: { + ok: boolean; + protocol?: number; + generation?: string; + result?: unknown; + error?: { code?: string; message?: string; delivery?: string }; + }; + try { + result = (await response.json()) as typeof result; + } catch { + throw failure( + "connection_lost", + "Desktop returned an incomplete response. Do not automatically repeat input.", + reads.has(method) ? "notDispatched" : "unknown", + ); + } + if (!result || typeof result !== "object") + throw failure( + "invalid_response", + "Invalid desktop response.", + reads.has(method) ? "notDispatched" : "unknown", + ); + if (!result.ok || !response.ok) + throw failure( + result.error?.code ?? "connection_failed", + result.error?.message ?? "Desktop request failed.", + result.error?.delivery ?? + (reads.has(method) ? "notDispatched" : "unknown"), + ); + if ( + result.protocol !== 1 || + (result.generation !== credential.generation && method !== "rebind") + ) + throw failure( + "generation_mismatch", + "Desktop protocol or lifecycle changed. Reconnect with a fresh credential.", + reads.has(method) ? "notDispatched" : "unknown", + ); + return result.result; +} diff --git a/src/desktop/controller.ts b/src/desktop/controller.ts index 6afe8e4..d82a7f3 100644 --- a/src/desktop/controller.ts +++ b/src/desktop/controller.ts @@ -1,35 +1,41 @@ -import { readFile, writeFile, rename } from "node:fs/promises"; +import { createHash, randomUUID } from "node:crypto"; +import { readFile, rename, writeFile } from "node:fs/promises"; import { join } from "node:path"; -import { randomUUID, createHash } from "node:crypto"; import { z } from "zod"; -import { X11Backend, type Geometry } from "./backend.js"; import { Authority, type Grant } from "./authority.js"; +import type { DesktopBackend, Geometry } from "./backend.js"; +import { Presence } from "./presence.js"; import { Action, - methodArguments, - Target, failure, + methodArguments, type Observation, type Request, scopes, + Target, } from "./protocol.js"; import { Recorder } from "./recording.js"; + type Lease = { id: string; grantId: string; + participantId?: string; subject: string; owner: "human" | "agent"; epoch: number; expiresAt: number; }; -type Frame = Awaited>; +type Frame = Awaited>; export class DesktopController { readonly authority: Authority; readonly recorder: Recorder; + readonly presence: Presence; private epoch = 1; private geometryKey = ""; private lease?: Lease; private fenced = false; + private closed = false; + private retryFenceAt = 0; private observations = new Map< string, { @@ -54,17 +60,23 @@ export class DesktopController { private timer: ReturnType; private recorderGrant?: string; constructor( - readonly backend: X11Backend, + readonly backend: DesktopBackend, generation: string, private directory: string, ) { this.authority = new Authority(generation); + this.presence = new Presence(this.authority); this.recorder = new Recorder(directory, (t) => this.capture(t)); this.timer = setInterval(() => { + this.presence.sweep(); + if (this.fenced && !this.fenceTask && Date.now() >= this.retryFenceAt) + void this.fence().catch(() => {}); if ( this.lease && (this.lease.expiresAt <= Date.now() || - !this.authority.valid(this.lease.grantId)) + !this.authority.valid(this.lease.grantId) || + (this.lease.participantId !== undefined && + !this.presence.has(this.lease.participantId))) ) void this.fence().catch(() => {}); if (this.recorderGrant && !this.authority.valid(this.recorderGrant)) { @@ -185,7 +197,21 @@ export class DesktopController { epoch: this.epoch, sample, }); - while (this.observations.size > 32) + // Bound each credential independently so one fast viewer cannot evict every + // other participant's observation before their input reaches the broker. + let grantCount = 0; + for (const record of this.observations.values()) + if (record.grantId === grant.id) grantCount++; + for (const [id, record] of this.observations) { + if ( + performance.now() - record.at > 30000 || + (record.grantId === grant.id && grantCount > 8) + ) { + this.observations.delete(id); + if (record.grantId === grant.id) grantCount--; + } + } + while (this.observations.size > 512) this.observations.delete(this.observations.keys().next().value!); return value; } @@ -193,47 +219,72 @@ export class DesktopController { async fence() { if (this.fenceTask) return this.fenceTask; this.fenced = true; + this.retryFenceAt = Date.now() + 1000; this.epoch++; this.lease = undefined; this.observations.clear(); this.active?.abort.abort(); const task = (async () => { await this.active?.done.catch(() => {}); - for (const key of this.heldKeys) - await this.backend - .input( - { kind: "display" }, - { kind: "keyUp", key }, - new AbortController().signal, - ) - .catch(() => {}); - for (const button of this.heldButtons) - await this.backend - .input( - { kind: "display" }, - { kind: "buttonUp", button }, - new AbortController().signal, - ) - .catch(() => {}); - this.heldKeys.clear(); - this.heldButtons.clear(); - await this.persistHeld(); + const abort = new AbortController(); + const deadline = setTimeout(() => abort.abort(), 10000); + try { + for (const key of this.heldKeys) { + if (abort.signal.aborted) break; + try { + await this.backend.input( + { kind: "display" }, + { kind: "keyUp", key }, + abort.signal, + ); + this.heldKeys.delete(key); + } catch { + /* Preserve failed releases in the recovery journal. */ + } + } + for (const button of this.heldButtons) { + if (abort.signal.aborted) break; + try { + await this.backend.input( + { kind: "display" }, + { kind: "buttonUp", button }, + abort.signal, + ); + this.heldButtons.delete(button); + } catch { + /* Preserve failed releases in the recovery journal. */ + } + } + await this.persistHeld(); + if (this.heldKeys.size || this.heldButtons.size) + throw failure( + "input_cleanup_failed", + "Held input could not be released. Control remains blocked until cleanup or service recovery succeeds.", + "unknown", + ); + this.fenced = false; + } finally { + clearTimeout(deadline); + } })(); this.fenceTask = task; try { await task; } finally { this.fenceTask = undefined; - this.fenced = false; } } private requireLease(grant: Grant, id: unknown) { if ( + this.closed || this.fenced || !this.lease || this.lease.id !== id || this.lease.grantId !== grant.id || - this.lease.expiresAt <= Date.now() + !this.authority.valid(grant.id) || + this.lease.expiresAt <= Date.now() || + (this.lease.participantId !== undefined && + !this.presence.has(this.lease.participantId)) ) throw failure( "lease_revoked", @@ -323,6 +374,8 @@ export class DesktopController { () => abort.abort(), Math.max(1, Math.min(10000, currentLease.expiresAt - Date.now())), ); + const previouslyHeldKeys = new Set(this.heldKeys); + const previouslyHeldButtons = new Set(this.heldButtons); if (action.kind === "keyDown") this.heldKeys.add(action.key); if (action.kind === "buttonDown") this.heldButtons.add(action.button); const transientButtons = ["click", "doubleClick", "drag"].includes( @@ -345,8 +398,10 @@ export class DesktopController { await this.persistHeld(); await this.backend.input(record.value.target, action, abort.signal); for (const button of transientButtons) - this.heldButtons.delete(button); - for (const key of transientKeys) this.heldKeys.delete(key); + if (!previouslyHeldButtons.has(button)) + this.heldButtons.delete(button); + for (const key of transientKeys) + if (!previouslyHeldKeys.has(key)) this.heldKeys.delete(key); if (action.kind === "keyUp") this.heldKeys.delete(action.key); if (action.kind === "buttonUp") this.heldButtons.delete(action.button); @@ -367,7 +422,7 @@ export class DesktopController { finish(); this.active = undefined; this.frame = undefined; - if (failed) await this.fence(); + if (failed) await this.fence().catch(() => {}); } }) .finally(() => this.queued--); @@ -375,14 +430,22 @@ export class DesktopController { return job; } async call(request: Request, grant: Grant, admin = false): Promise { + if (this.closed) + throw failure("service_closed", "Desktop service is closing."); if (request.generation !== this.generation) throw failure("generation_mismatch", "Host rebind required."); + if (!admin && !this.authority.valid(grant.id)) + throw failure("permission_denied", "Credential expired or revoked."); const methods: Record = { health: "observe", apps: "observe", windows: "observe", observe: "observe", state: "viewer-read", + "presence.list": "viewer-read", + "presence.join": "presence", + "presence.update": "presence", + "presence.leave": "presence", acquire: "input-control", renew: "input-control", release: "input-control", @@ -407,6 +470,8 @@ export class DesktopController { "observe", "health", "state", + "presence.list", + "presence.update", "windows", "apps", "recording.list", @@ -441,16 +506,24 @@ export class DesktopController { displayId: this.backend.env.DISPLAY ?? "unknown", generation: this.generation, displayEpoch: this.epoch, - backend: "x11", + backend: this.backend.capabilities?.backend ?? "x11", mode: "attach", geometry: frame.geometry, rawCapture: true, perception: false, recording: await this.recorder.available(), input: true, - resize: "randr-dependent", + multiplayer: { + presence: true, + cursorSpace: "normalized-target", + maxParticipants: 64, + heartbeatMs: 5000, + participantTtlMs: 15000, + inputOwners: 1, + }, + resize: this.backend.capabilities?.resize ?? "randr-dependent", coordinateSpace: "image-pixels", - limitations: [ + limitations: this.backend.capabilities?.limitations ?? [ "No Wayland, audio or synchronized multi-monitor.", "External X clients bypass application arbitration.", ], @@ -465,15 +538,30 @@ export class DesktopController { return this.backend.windows(); case "observe": return this.observe(g, Target.parse(a.target ?? { kind: "display" })); + case "presence.join": + return this.presence.join( + g, + a.name as string | undefined, + a.role as "human" | "agent", + ); + case "presence.update": + return this.presence.update( + g, + a.participantId as string, + a.cursor as { x: number; y: number } | null | undefined, + ); + case "presence.leave": { + const id = a.participantId as string; + this.presence.leave(g, id); + if (this.lease?.participantId === id) await this.fence(); + return { left: true }; + } + case "presence.list": + return this.multiplayerState(); case "state": return { - lease: this.lease - ? { - owner: this.lease.owner, - subject: this.lease.subject, - expiresAt: this.lease.expiresAt, - } - : null, + ...this.multiplayerState(), + lease: this.controllerState(), displayEpoch: this.epoch, recordings: this.recorder .list() @@ -481,6 +569,8 @@ export class DesktopController { }; case "acquire": case "takeover": { + const participantId = a.participantId as string | undefined; + if (participantId) this.presence.require(g, participantId); if (this.fenced) throw failure("lease_conflict", "Control transition is in progress."); if ( @@ -490,12 +580,14 @@ export class DesktopController { ) throw failure("lease_conflict", "Display already has a controller."); await this.fence(); - if (!this.authority.valid(g.id) && !admin) + if (this.closed || (!this.authority.valid(g.id) && !admin)) throw failure( "permission_denied", "Credential expired during takeover.", ); + if (participantId) this.presence.require(g, participantId); this.lease = { + participantId, id: randomUUID(), grantId: g.id, subject: g.subject, @@ -571,6 +663,7 @@ export class DesktopController { await this.fence(); await this.recorder.fence(); this.authority.rebind(next); + this.presence.clear(); this.frame = undefined; this.cache.clear(); return { generation: next }; @@ -623,12 +716,32 @@ export class DesktopController { } throw failure("unsupported", "Unsupported desktop operation."); } + private controllerState() { + if ( + !this.lease || + this.lease.expiresAt <= Date.now() || + !this.authority.valid(this.lease.grantId) || + (this.lease.participantId !== undefined && + !this.presence.has(this.lease.participantId)) + ) + return null; + const { owner, subject, participantId, expiresAt } = this.lease; + return { owner, subject, participantId, expiresAt }; + } + multiplayerState() { + return { + participants: this.presence.list(), + controller: this.controllerState(), + }; + } async cancelGrantInput(grantId: string) { if (this.lease?.grantId === grantId) await this.fence(); } async close() { + if (this.closed) return; + this.closed = true; clearInterval(this.timer); - await this.fence(); - await this.recorder.fence(); + this.presence.clear(); + await Promise.allSettled([this.fence(), this.recorder.fence()]); } } diff --git a/src/desktop/index.ts b/src/desktop/index.ts index 8bac0b6..f92496f 100644 --- a/src/desktop/index.ts +++ b/src/desktop/index.ts @@ -1,11 +1,19 @@ -export { Config, startDesktop, registryPath } from "./server.js"; +export type { Credential } from "./client.js"; +export { + desktopCall, + readCredential, + serviceURL, + writeCredential, +} from "./client.js"; +export type { Observation, Participant, Scope } from "./protocol.js"; export { - Request, Action, - Target, - scopes, - VERSION, methodArguments, methodSchema, + Request, + scopes, + Target, + VERSION, } from "./protocol.js"; -export type { Observation, Scope } from "./protocol.js"; +export { openSSHTunnel } from "./remote.js"; +export { Config, registryPath, startDesktop } from "./server.js"; diff --git a/src/desktop/mac-backend.ts b/src/desktop/mac-backend.ts new file mode 100644 index 0000000..586c862 --- /dev/null +++ b/src/desktop/mac-backend.ts @@ -0,0 +1,144 @@ +import { z } from "zod"; +import type { DesktopBackend, Geometry } from "./backend.js"; +import { type Action, failure, type Target } from "./protocol.js"; + +export interface MacDriver { + request( + method: string, + args?: Record, + signal?: AbortSignal, + ): Promise; + close?(): void; +} +const GeometrySchema = z.object({ + id: z.number().int().positive(), + x: z.number().finite(), + y: z.number().finite(), + width: z.number().int().positive().max(8192), + height: z.number().int().positive().max(8192), + pid: z.number().int().positive().optional(), + title: z.string().optional(), +}); +export class MacBackend implements DesktopBackend { + readonly env = { DISPLAY: "macos" }; + readonly capabilities = { + backend: "macos", + resize: false as const, + limitations: [ + "Requires macOS 14+, Screen Recording and Accessibility permissions for Opcode.", + "Primary display only; physical input is serialized, participant cursors are overlays.", + "Local applications and physical input bypass broker arbitration.", + ], + }; + private supportedKeys?: Promise>; + constructor(readonly driver: MacDriver) {} + async geometry(target: Target): Promise { + return GeometrySchema.parse( + await this.driver.request("desktopGeometry", { target }), + ); + } + async windows(): Promise { + return z + .array(GeometrySchema) + .parse(await this.driver.request("desktopWindows")); + } + async apps() { + return z + .array(z.object({ id: z.string(), name: z.string() })) + .parse(await this.driver.request("installedApps")); + } + async state() { + return z + .object({ + focus: z.number().int(), + x: z.number().finite(), + y: z.number().finite(), + }) + .parse(await this.driver.request("desktopState")); + } + async capture(target: Target) { + const result = z + .object({ + geometry: GeometrySchema, + base64: z.string().max(16_000_000), + sample: z.string().max(16_384), + }) + .parse(await this.driver.request("desktopCapture", { target })); + const png = Buffer.from(result.base64, "base64"); + const sample = Buffer.from(result.sample, "base64"); + if ( + png.length < 24 || + !png + .subarray(0, 8) + .equals(Buffer.from([137, 80, 78, 71, 13, 10, 26, 10])) || + png.readUInt32BE(16) !== result.geometry.width || + png.readUInt32BE(20) !== result.geometry.height || + sample.length !== 64 * 48 * 4 + ) + throw failure( + "display_unavailable", + "Mac helper returned an invalid desktop frame.", + ); + return { geometry: result.geometry, png, ppm: Buffer.alloc(0), sample }; + } + async prepareAction(action: Action) { + if ( + action.kind === "key" || + action.kind === "keyDown" || + action.kind === "keyUp" + ) { + // Query the native source of truth before the controller journals held input. + this.supportedKeys ??= this.driver + .request("desktopKeys") + .then( + (value) => + new Set( + z.array(z.string().min(1).max(100)).min(1).max(256).parse(value), + ), + ); + const supported = await this.supportedKeys; + const parts = action.key.split("+"); + const modifiers = new Set(["Meta", "Control", "Alt", "Shift"]); + if ( + parts.some( + (key) => + !supported.has(key) && + !(key.length === 1 && supported.has(key.toLowerCase())), + ) || + (action.kind !== "key" && parts.length !== 1) || + parts.slice(0, -1).some((key) => !modifiers.has(key)) + ) + throw failure( + "unsupported", + "Unsupported Mac key or modifier combination.", + ); + } + return action; + } + async input(target: Target, action: Action, signal: AbortSignal) { + signal.throwIfAborted(); + if (action.kind === "launch") { + if (!(await this.apps()).some((app) => app.id === action.appId)) + throw failure("not_found", "Unknown installed app ID."); + signal.throwIfAborted(); + await this.driver.request("launchApp", { appId: action.appId }, signal); + return; + } + // Releases must remain possible after the originally observed window disappears. + if (action.kind === "keyUp" || action.kind === "buttonUp") + target = { kind: "display" }; + const geometry = await this.geometry(target); + signal.throwIfAborted(); + await this.driver.request( + "desktopInput", + { target, action, expectedGeometry: geometry, foregroundApproved: true }, + signal, + ); + } + async resize(_width: number, _height: number): Promise { + throw failure( + "unsupported", + "Resizing a physical Mac display is not supported.", + ); + } +} diff --git a/src/desktop/mac-driver.ts b/src/desktop/mac-driver.ts new file mode 100644 index 0000000..ee00106 --- /dev/null +++ b/src/desktop/mac-driver.ts @@ -0,0 +1,164 @@ +import { randomUUID } from "node:crypto"; +import { lstat } from "node:fs/promises"; +import { connect, type Socket } from "node:net"; +import type { MacDriver } from "./mac-backend.js"; +import { failure } from "./protocol.js"; + +/** Connect only to the installed LaunchServices helper; never spawn an untrusted binary. */ +export async function connectMacDriver(): Promise { + const uid = process.getuid?.(); + if (uid === undefined) + throw failure("unsupported", "Mac helper requires a Unix user."); + const directory = `/tmp/opcode-cu-${uid}`; + for (const path of [directory, `${directory}/driver.sock`]) { + const stat = await lstat(path).catch(() => { + throw failure( + "display_unavailable", + "Start the installed Opcode helper with cu install, then grant its permissions.", + ); + }); + if ( + stat.isSymbolicLink() || + stat.uid !== uid || + stat.mode & 0o077 || + (path === directory ? !stat.isDirectory() : !stat.isSocket()) + ) + throw failure( + "permission_denied", + "Unsafe Mac helper socket permissions.", + ); + } + const socket = await new Promise((resolve, reject) => { + const socket = connect(`${directory}/driver.sock`); + const timeout = setTimeout( + () => socket.destroy(new Error("Mac helper connection timed out.")), + 3000, + ); + socket.once("error", reject); + socket.once("connect", () => { + clearTimeout(timeout); + socket.off("error", reject); + resolve(socket); + }); + socket.once("close", () => clearTimeout(timeout)); + }); + return socketMacDriver(socket); +} + +/** Exported for transport tests. A disconnected session never reconnects or replays input. */ +export function socketMacDriver(socket: Socket): MacDriver { + let chunks: Buffer[] = []; + let bufferedBytes = 0; + let closed = false; + let tail: Promise = Promise.resolve(); + let pending: + | { + id: string; + resolve(value: unknown): void; + reject(error: Error): void; + timer: ReturnType; + } + | undefined; + const disconnect = () => { + closed = true; + if (pending) { + clearTimeout(pending.timer); + pending.reject( + failure( + "driver_disconnected", + "Mac helper disconnected. Observe before retrying; input delivery is unknown.", + "unknown", + ), + ); + pending = undefined; + } + socket.destroy(); + }; + socket.on("error", disconnect); + socket.on("close", disconnect); + socket.on("data", (chunk: Buffer) => { + let offset = 0; + while (offset < chunk.length) { + const index = chunk.indexOf(10, offset); + const part = chunk.subarray(offset, index < 0 ? chunk.length : index); + chunks.push(part); + bufferedBytes += part.length; + if (bufferedBytes > 17_000_000) { + disconnect(); + return; + } + if (index < 0) return; + const line = Buffer.concat(chunks, bufferedBytes); + chunks = []; + bufferedBytes = 0; + offset = index + 1; + try { + const reply = JSON.parse(line.toString()); + if ( + !pending || + reply.id !== pending.id || + typeof reply.ok !== "boolean" + ) { + disconnect(); + return; + } + const task = pending; + pending = undefined; + clearTimeout(task.timer); + if (reply.ok) task.resolve(reply.data); + else + task.reject( + failure( + reply.error?.code ?? "driver_error", + reply.error?.message ?? "Mac helper request failed.", + reply.error?.delivery ?? "unknown", + ), + ); + } catch { + disconnect(); + return; + } + } + }); + return { + close: disconnect, + request(method, args = {}, signal) { + const task = tail + .catch(() => {}) + .then(async () => { + signal?.throwIfAborted(); + if (closed) + throw failure( + "driver_disconnected", + "Mac helper is disconnected. Restart the desktop service.", + ); + const id = randomUUID(); + const wire = JSON.stringify({ ...args, id, method }) + "\n"; + if (Buffer.byteLength(wire) > 128_000) + throw failure( + "invalid_request", + "Mac helper request exceeds limit.", + ); + const result = await new Promise((resolve, reject) => { + pending = { + id, + resolve, + reject, + timer: setTimeout(disconnect, 12_000), + }; + socket.write(wire); + }); + // Wait for completion before fencing a cancelled in-flight action. Never retry it. + if (signal?.aborted) + throw failure( + "cancelled", + "Mac operation finished after cancellation; verify fresh state.", + "unknown", + ); + return result; + }); + tail = task; + return task; + }, + }; +} diff --git a/src/desktop/presence.ts b/src/desktop/presence.ts new file mode 100644 index 0000000..3cbd0c4 --- /dev/null +++ b/src/desktop/presence.ts @@ -0,0 +1,104 @@ +import { randomUUID } from "node:crypto"; +import type { Authority, Grant } from "./authority.js"; +import { failure, type Participant } from "./protocol.js"; + +/** Visual cursors are presence only. Actual OS input remains lease-controlled. */ +export class Presence { + private entries = new Map< + string, + { grantId: string; value: Participant; updatedAt: number } + >(); + constructor( + private authority: Authority, + private now = () => Date.now(), + ) {} + sweep() { + const removed: string[] = []; + for (const [id, entry] of this.entries) { + if ( + entry.value.expiresAt <= this.now() || + !this.authority.valid(entry.grantId) + ) { + this.entries.delete(id); + removed.push(id); + } + } + return removed; + } + list(): Participant[] { + this.sweep(); + return [...this.entries.values()].map(({ value }) => + structuredClone(value), + ); + } + has(id: string) { + this.sweep(); + return this.entries.has(id); + } + require(grant: Grant, id: string) { + this.sweep(); + const entry = this.entries.get(id); + if (!entry || entry.grantId !== grant.id) + throw failure( + "participant_expired", + "Participant expired or belongs to another credential; join again.", + ); + return entry; + } + join( + grant: Grant, + name: string | undefined, + role: Participant["role"], + ): Participant { + this.sweep(); + if ( + this.entries.size >= 64 || + [...this.entries.values()].filter((e) => e.grantId === grant.id).length >= + 8 + ) + throw failure("overloaded", "Participant limit reached."); + const id = randomUUID(); + const colors = [ + "#60a5fa", + "#f472b6", + "#34d399", + "#fbbf24", + "#a78bfa", + "#fb923c", + ]; + const value: Participant = { + id, + subject: grant.subject, + name: name ?? grant.subject.slice(0, 64), + role, + color: colors[parseInt(id.slice(0, 8), 16) % colors.length] ?? "#60a5fa", + cursor: null, + expiresAt: Math.min(grant.expiresAt, this.now() + 15000), + }; + this.entries.set(id, { grantId: grant.id, value, updatedAt: -Infinity }); + return structuredClone(value); + } + update( + grant: Grant, + id: string, + cursor: Participant["cursor"] | undefined, + ): Participant { + const entry = this.require(grant, id); + if (this.now() - entry.updatedAt < 40) + throw failure( + "overloaded", + "Presence updates are limited to 25 per second.", + ); + entry.updatedAt = this.now(); + if (cursor !== undefined) entry.value.cursor = cursor; + entry.value.expiresAt = Math.min(grant.expiresAt, this.now() + 15000); + return structuredClone(entry.value); + } + leave(grant: Grant, id: string) { + this.require(grant, id); + this.entries.delete(id); + } + clear() { + this.entries.clear(); + } +} diff --git a/src/desktop/protocol.ts b/src/desktop/protocol.ts index ab47404..d2d9591 100644 --- a/src/desktop/protocol.ts +++ b/src/desktop/protocol.ts @@ -3,6 +3,7 @@ export const VERSION = 1; export const scopes = [ "observe", "viewer-read", + "presence", "input-control", "recording-start", "recording-read", @@ -61,7 +62,11 @@ export const Action = z.discriminatedUnion("kind", [ }), z.object({ kind: z.literal("launch"), - appId: z.string().regex(/^[a-zA-Z0-9_.-]+\.desktop$/), + appId: z + .string() + .min(1) + .max(256) + .regex(/^[a-zA-Z0-9_.-]+$/), }), z.object({ kind: z.literal("focus"), windowId: z.number().int().positive() }), ]); @@ -108,6 +113,21 @@ export interface Observation { } const leaseId = z.string().min(1); +export const Cursor = z.object({ + target: Target.optional(), + x: z.number().finite().min(0).max(1), + y: z.number().finite().min(0).max(1), +}); +export interface Participant { + id: string; + subject: string; + name: string; + role: "human" | "agent"; + color: string; + cursor: z.infer | null; + expiresAt: number; +} +const participantId = z.string().uuid(); const recordingId = z.object({ id: z.string().uuid() }); /** Tool metadata and runtime argument validation share these definitions. */ export const methodArguments = { @@ -116,10 +136,22 @@ export const methodArguments = { windows: z.object({}), state: z.object({}), observe: z.object({ target: Target.default({ kind: "display" }) }), + "presence.join": z.object({ + name: z.string().trim().min(1).max(64).optional(), + role: z.enum(["human", "agent"]).default("human"), + }), + "presence.update": z.object({ + participantId, + cursor: Cursor.nullable().optional(), + }), + "presence.leave": z.object({ participantId }), + "presence.list": z.object({}), acquire: z.object({ + participantId: participantId.optional(), ttlMs: z.number().int().min(1000).max(60000).default(30000), }), takeover: z.object({ + participantId: participantId.optional(), ttlMs: z.number().int().min(1000).max(60000).default(30000), }), renew: z.object({ leaseId }), diff --git a/src/desktop/registry.ts b/src/desktop/registry.ts new file mode 100644 index 0000000..5e153ca --- /dev/null +++ b/src/desktop/registry.ts @@ -0,0 +1,25 @@ +import { createHash } from "node:crypto"; +import { homedir } from "node:os"; +import { join } from "node:path"; +import { + canonical as canonicalX11, + displayKey as x11Key, + registryPath as x11Path, +} from "../linux/display-registry.js"; +export const canonical = (display: string) => + display === "macos" ? display : canonicalX11(display); +export const displayKey = (display: string) => + display === "macos" + ? createHash("sha256") + .update(`${process.getuid?.()}:macos`) + .digest("hex") + .slice(0, 24) + : x11Key(display); +export const registryPath = (display: string) => + display === "macos" + ? join( + homedir(), + ".local/state/opcode/displays", + displayKey(display) + ".json", + ) + : x11Path(display); diff --git a/src/desktop/remote.ts b/src/desktop/remote.ts new file mode 100644 index 0000000..b5ccce8 --- /dev/null +++ b/src/desktop/remote.ts @@ -0,0 +1,207 @@ +import { execFile, spawn } from "node:child_process"; +import { unlink } from "node:fs/promises"; +import { promisify } from "node:util"; +import { + Credential, + desktopCall, + serviceURL, + writeCredential, +} from "./client.js"; +import { failure } from "./protocol.js"; + +const exec = promisify(execFile); + +export function sshTarget(value: string) { + if (!/^(?:[a-zA-Z0-9_.-]+@)?[a-zA-Z0-9][a-zA-Z0-9_.-]*$/.test(value)) + throw failure( + "invalid_request", + "Use an SSH config alias or user@hostname.", + ); + return value; +} +export function remoteCredentialCommand(path: string) { + if (!path.startsWith("/") || path.length > 4096 || /[\0\r\n]/.test(path)) + throw failure( + "invalid_request", + "Remote credential path must be absolute.", + ); + return "cat -- '" + path.replaceAll("'", "'\\''") + "'"; +} + +/** OpenSSH retains ownership of keys, host verification, proxies and Tailnet routing. */ +export async function openSSHTunnel(options: { + host: string; + remoteCredentialFile: string; + output: string; + port: number; +}) { + const host = sshTarget(options.host); + if ( + !Number.isInteger(options.port) || + options.port < 1024 || + options.port > 65535 + ) + throw failure( + "invalid_request", + "Local port must be between 1024 and 65535.", + ); + const common = [ + "-T", + "-o", + "BatchMode=yes", + "-o", + "ConnectTimeout=10", + "-o", + "StrictHostKeyChecking=yes", + ]; + let remote: Credential; + try { + const { stdout } = await exec( + "ssh", + [ + ...common, + "--", + host, + remoteCredentialCommand(options.remoteCredentialFile), + ], + { timeout: 15000, maxBuffer: 16384 }, + ); + remote = Credential.parse(JSON.parse(stdout)); + } catch { + throw failure( + "connection_failed", + "Could not read the scoped remote credential. Check the SSH alias, accepted host key, non-interactive authentication and absolute credential path.", + ); + } + const endpoint = serviceURL(remote.endpoint); + if ( + endpoint.protocol !== "http:" || + !["localhost", "127.0.0.1"].includes(endpoint.hostname) + ) + throw failure( + "invalid_request", + "SSH sharing requires a remote loopback HTTP credential. For HTTPS, use the credential directly.", + ); + const port = endpoint.port || "80"; + const local: Credential = { + ...remote, + endpoint: `http://127.0.0.1:${options.port}${endpoint.pathname.replace(/\/$/, "")}`, + }; + const child = spawn( + "ssh", + [ + ...common, + "-N", + "-o", + "PermitLocalCommand=yes", + "-o", + "LocalCommand=printf OPCODE_TUNNEL_READY", + "-o", + "ExitOnForwardFailure=yes", + "-o", + "ServerAliveInterval=10", + "-o", + "ServerAliveCountMax=3", + "-L", + `127.0.0.1:${options.port}:127.0.0.1:${port}`, + "--", + host, + ], + { stdio: ["ignore", "pipe", "ignore"] }, + ); + let exited = false; + const readySignal = new Promise((resolve, reject) => { + let output = ""; + const timer = setTimeout( + () => + reject( + failure( + "connection_failed", + "SSH did not confirm forwarding readiness.", + ), + ), + 15000, + ); + const finish = () => { + clearTimeout(timer); + }; + child.stdout?.on("data", (chunk) => { + output += chunk.toString(); + if (output.includes("OPCODE_TUNNEL_READY")) { + finish(); + resolve(); + } else if (output.length > 4096) { + finish(); + reject(failure("connection_failed", "Unexpected SSH startup output.")); + } + }); + child.once("error", () => { + finish(); + reject(failure("connection_failed", "SSH could not start.")); + }); + child.once("exit", () => { + finish(); + reject( + failure( + "connection_failed", + "SSH closed before forwarding became ready.", + ), + ); + }); + }); + let written = false; + let cleaned = false; + const done = new Promise((resolve) => { + child.once("error", () => { + exited = true; + resolve(); + }); + child.once("exit", () => { + exited = true; + resolve(); + }); + }); + const close = async () => { + if (cleaned) return; + cleaned = true; + child.kill("SIGTERM"); + const timer = setTimeout(() => child.kill("SIGKILL"), 1000); + timer.unref(); + await done; + clearTimeout(timer); + if (written) await unlink(options.output).catch(() => {}); + }; + try { + await readySignal; + let ready = false; + for (let i = 0; i < 50 && !exited; i++) { + try { + await desktopCall( + local, + "presence.list", + {}, + { signal: AbortSignal.timeout(1000) }, + ); + ready = true; + break; + } catch (error) { + if ((error as { code?: string }).code !== "connection_lost") + throw error; + } + await new Promise((resolve) => setTimeout(resolve, 100)); + } + if (!ready || exited) + throw failure( + "connection_failed", + "SSH tunnel did not become ready. Verify the desktop broker, grant expiry, and local port availability.", + ); + await writeCredential(options.output, local); + written = true; + if (exited) + throw failure("connection_lost", "SSH tunnel closed during setup."); + return { endpoint: local.endpoint, close, done }; + } catch (error) { + await close(); + throw error; + } +} diff --git a/src/desktop/server.ts b/src/desktop/server.ts index 9f17c34..fbbc888 100644 --- a/src/desktop/server.ts +++ b/src/desktop/server.ts @@ -1,29 +1,31 @@ -import { createServer as httpServer } from "node:http"; -import { createServer as socketServer } from "node:net"; -import { randomBytes, createHash, timingSafeEqual } from "node:crypto"; +import { createHash, randomBytes, timingSafeEqual } from "node:crypto"; +import { createReadStream } from "node:fs"; import { - mkdir, - writeFile, chmod, - stat, + lstat, + mkdir, readFile, + stat, unlink, - lstat, + writeFile, } from "node:fs/promises"; -import { createReadStream } from "node:fs"; -import { dirname, join } from "node:path"; +import { createServer as httpServer, type Server } from "node:http"; +import { createServer as socketServer } from "node:net"; import { homedir } from "node:os"; +import { dirname, join } from "node:path"; import { z } from "zod"; import { X11Backend } from "./backend.js"; import { DesktopController } from "./controller.js"; -import { Target, Request, scopes, errorJSON, failure } from "./protocol.js"; +import { MacBackend, type MacDriver } from "./mac-backend.js"; +import { connectMacDriver } from "./mac-driver.js"; +import { errorJSON, failure, Request, scopes, Target } from "./protocol.js"; import { desktopHTML } from "./view.js"; export const Config = z.object({ - display: z.string().regex(/^(?:unix)?:(\d+)(?:\.(\d+))?$/), + display: z.string().regex(/^(?:macos|(?:unix)?:(\d+)(?:\.(\d+))?)$/), authority: z.string().optional(), uid: z.number().int().nonnegative(), generation: z.string().min(1).max(128), - helper: z.string(), + helper: z.string().default(""), directory: z.string().optional(), origin: z.string().url().optional(), basePath: z @@ -32,25 +34,25 @@ export const Config = z.object({ .default(""), }); export type Config = z.infer; -export { registryPath } from "../linux/display-registry.js"; -import { - canonical, - displayKey, - registryPath, -} from "../linux/display-registry.js"; +export { registryPath } from "./registry.js"; + +import { canonical, displayKey, registryPath } from "./registry.js"; export async function startDesktop(input: z.input) { const config = Config.parse(input); - if (process.platform !== "linux") + const mac = config.display === "macos"; + if (mac ? process.platform !== "darwin" : process.platform !== "linux") throw failure( "unsupported", - "Display service currently requires Linux X11.", + "Use display macos on macOS, or a local X11 display on Linux.", ); + if (!mac && !config.helper) + throw failure("invalid_request", "Linux requires an X11 helper path."); if (config.uid !== process.getuid?.()) throw failure( "permission_denied", "Configured display user differs from the service user.", ); - if (config.authority) + if (!mac && config.authority) await stat(config.authority).catch(() => { throw failure("permission_denied", "Xauthority file is unavailable."); }); @@ -66,288 +68,349 @@ export async function startDesktop(input: z.input) { ), ), ); - lock.listen( - "\0opcode-desktop-" + - createHash("sha256").update(display).digest("hex").slice(0, 24), - resolve, - ); - }); - await mkdir(dirname(path), { recursive: true, mode: 0o700 }); - const directoryInfo = await lstat(dirname(path)); - if ( - !directoryInfo.isDirectory() || - directoryInfo.isSymbolicLink() || - directoryInfo.uid !== process.getuid?.() || - directoryInfo.mode & 0o077 - ) { - lock.close(); - throw failure("permission_denied", "Unsafe broker directory."); - } - const directory = - config.directory ?? - join(homedir(), ".local/state/opcode/recordings", displayKey(display)); - const backend = new X11Backend(config.helper, { - ...process.env, - DISPLAY: display, - ...(config.authority ? { XAUTHORITY: config.authority } : {}), + // Linux keeps its namespace-wide abstract X11 reservation. Mac ownership is + // acquired below through the persistent native helper connection and flock. + if (mac) resolve(); + else + lock.listen( + "\0opcode-desktop-" + + createHash("sha256").update(display).digest("hex").slice(0, 24), + resolve, + ); }); - const controller = new DesktopController( - backend, - config.generation, - directory, - ); - let closed = false; + let startupController: DesktopController | undefined; + let startupDriver: MacDriver | undefined; + let startupServer: Server | undefined; try { - await controller.init(); - } catch (e) { - lock.close(); - await controller.close(); - throw e; - } - const master = randomBytes(32).toString("hex"); - let host = controller.authority.issue("host", [...scopes], 3600000); - let inFlight = 0; - const base = config.basePath; - const authenticate = ( - header: string | undefined, - scope?: (typeof scopes)[number], - ) => { - const token = header?.replace(/^Bearer /, "") ?? ""; - const admin = - token.length === master.length && - timingSafeEqual(Buffer.from(token), Buffer.from(master)); - if (admin) { - try { - controller.authority.verify(host.token); - } catch { - host = controller.authority.issue("host", [...scopes], 3600000); - } - return { grant: controller.authority.verify(host.token, scope), admin }; + await mkdir(dirname(path), { recursive: true, mode: 0o700 }); + const directoryInfo = await lstat(dirname(path)); + if ( + !directoryInfo.isDirectory() || + directoryInfo.isSymbolicLink() || + directoryInfo.uid !== process.getuid?.() || + directoryInfo.mode & 0o077 + ) { + lock.close(); + throw failure("permission_denied", "Unsafe broker directory."); } - return { grant: controller.authority.verify(token, scope), admin: false }; - }; - const server = httpServer(async (req, res) => { - res.setHeader("Cache-Control", "no-store"); - res.setHeader("Referrer-Policy", "no-referrer"); - res.setHeader("X-Content-Type-Options", "nosniff"); - const send = (status: number, value: unknown) => { - res.writeHead(status, { "Content-Type": "application/json" }); - res.end(JSON.stringify(value)); - }; - let counted = false; - const finished = () => { - if (counted) { - counted = false; - inFlight--; - } - }; - res.once("finish", finished); - res.once("close", finished); - try { - const expected = config.origin ?? origin; - if (req.headers.origin && req.headers.origin !== expected) - throw failure("permission_denied", "Origin is not allowed."); - if (req.headers.origin === expected) { - res.setHeader("Access-Control-Allow-Origin", expected); - res.setHeader("Vary", "Origin"); - } - const url = new URL(req.url ?? "/", origin); - const route = url.pathname; - if (route === base + "/view" && req.method === "GET") { - res.setHeader("Content-Type", "text/html; charset=utf-8"); - res.setHeader( - "Content-Security-Policy", - "default-src 'none'; script-src 'unsafe-inline'; style-src 'unsafe-inline'; img-src data:; connect-src 'self'; frame-ancestors " + - (config.origin ?? "'self'"), + const directory = + config.directory ?? + join(homedir(), ".local/state/opcode/recordings", displayKey(display)); + const macDriver = mac ? await connectMacDriver() : undefined; + startupDriver = macDriver; + if (macDriver) { + z.object({ locked: z.literal(true) }).parse( + await macDriver.request("desktopLock"), + ); + const status = z + .object({ accessibility: z.boolean(), screenRecording: z.boolean() }) + .parse(await macDriver.request("status")); + if (!status.accessibility || !status.screenRecording) + throw failure( + "permission_denied", + "Enable Accessibility and Screen Recording for the installed Opcode helper, then attach again.", ); - res.end(desktopHTML(base)); - return; - } - if (req.method === "OPTIONS") { - res.writeHead(204, { - "Access-Control-Allow-Headers": "Authorization, Content-Type", - "Access-Control-Allow-Methods": "GET, POST, DELETE, OPTIONS", - }); - res.end(); - return; - } - const { grant, admin } = authenticate(req.headers.authorization); - if (route === base + "/session") { - send(200, { - generation: controller.generation, - protocol: 1, - scopes: grant.scopes, + } + const backend = macDriver + ? new MacBackend(macDriver) + : new X11Backend(config.helper, { + ...process.env, + DISPLAY: display, + ...(config.authority ? { XAUTHORITY: config.authority } : {}), }); - return; + const controller = new DesktopController( + backend, + config.generation, + directory, + ); + startupController = controller; + let closed = false; + await controller.init(); + const master = randomBytes(32).toString("hex"); + let host = controller.authority.issue("host", [...scopes], 3600000); + let inFlight = 0; + const base = config.basePath; + const authenticate = ( + header: string | undefined, + scope?: (typeof scopes)[number], + ) => { + const token = header?.replace(/^Bearer /, "") ?? ""; + const admin = + token.length === master.length && + timingSafeEqual(Buffer.from(token), Buffer.from(master)); + if (admin) { + try { + controller.authority.verify(host.token); + } catch { + host = controller.authority.issue("host", [...scopes], 3600000); + } + return { grant: controller.authority.verify(host.token, scope), admin }; } - if (route === base + "/frame" && req.method === "GET") { - authenticate(req.headers.authorization, "viewer-read"); - if (inFlight >= 32) - throw failure("overloaded", "Too many concurrent requests."); + return { grant: controller.authority.verify(token, scope), admin: false }; + }; + const server = httpServer(async (req, res) => { + res.setHeader("Cache-Control", "no-store"); + res.setHeader("Referrer-Policy", "no-referrer"); + res.setHeader("X-Content-Type-Options", "nosniff"); + const send = (status: number, value: unknown) => { + res.writeHead(status, { "Content-Type": "application/json" }); + res.end(JSON.stringify(value)); + }; + let counted = false; + const finished = () => { + if (counted) { + counted = false; + inFlight--; + } + }; + res.once("finish", finished); + res.once("close", finished); + try { + const expected = req.headers.origin; + const allowed = + !expected || expected === origin || expected === config.origin; + if (!allowed) + throw failure("permission_denied", "Origin is not allowed."); + if (expected && allowed) { + res.setHeader("Access-Control-Allow-Origin", expected); + res.setHeader("Vary", "Origin"); + } + const url = new URL(req.url ?? "/", origin); + const route = url.pathname; + if (route === base + "/view" && req.method === "GET") { + res.setHeader("Content-Type", "text/html; charset=utf-8"); + res.setHeader( + "Content-Security-Policy", + "default-src 'none'; script-src 'unsafe-inline'; style-src 'unsafe-inline'; img-src data:; connect-src 'self'; frame-ancestors " + + (config.origin ?? "'self'"), + ); + res.end(desktopHTML(base)); + return; + } + if (req.method === "OPTIONS") { + res.writeHead(204, { + "Access-Control-Allow-Headers": "Authorization, Content-Type", + "Access-Control-Allow-Methods": "GET, POST, DELETE, OPTIONS", + }); + res.end(); + return; + } + const { grant, admin } = authenticate(req.headers.authorization); + if (route === base + "/session") { + send(200, { + generation: controller.generation, + protocol: 1, + scopes: grant.scopes, + admin, + subject: grant.subject, + }); + return; + } + if (route === base + "/frame" && req.method === "GET") { + authenticate(req.headers.authorization, "viewer-read"); + if (inFlight >= 32) + throw failure("overloaded", "Too many concurrent requests."); + inFlight++; + counted = true; + const windowId = url.searchParams.get("windowId"); + const target = Target.parse( + windowId === null + ? { kind: "display" } + : { kind: "window", id: Number(windowId) }, + ); + const frame = await controller.observe(grant, target); + authenticate(req.headers.authorization, "viewer-read"); + send(200, { + ...frame, + ...controller.multiplayerState(), + recording: controller.recorder + .list() + .filter((r) => r.state === "recording" || r.state === "paused") + .map(({ id, state }) => ({ id, state })), + }); + return; + } + if (route.startsWith(base + "/artifacts/") && req.method === "GET") { + authenticate(req.headers.authorization, "recording-read"); + const id = route.slice((base + "/artifacts/").length); + if (!/^[a-f0-9-]{36}$/.test(id)) + throw failure("not_found", "Artifact not found."); + const file = controller.recorder.path(id), + info = await stat(file); + let start = 0, + end = info.size - 1; + if (req.headers.range) { + const match = /^bytes=(\d+)-(\d*)$/.exec(req.headers.range); + if (!match) + throw failure("invalid_range", "Use one explicit byte range."); + start = +match[1]; + end = match[2] ? +match[2] : end; + if (start > end || end >= info.size) { + res.writeHead(416, { "Content-Range": `bytes */${info.size}` }); + res.end(); + return; + } + } + res.writeHead(req.headers.range ? 206 : 200, { + "Content-Type": "video/mp4", + "Accept-Ranges": "bytes", + "Content-Length": end - start + 1, + "Content-Disposition": `attachment; filename="${id}.mp4"`, + ...(req.headers.range + ? { "Content-Range": `bytes ${start}-${end}/${info.size}` } + : {}), + }); + const stream = createReadStream(file, { start, end }); + const timer = setInterval(() => { + try { + authenticate(req.headers.authorization, "recording-read"); + } catch { + stream.destroy(); + res.destroy(); + } + }, 100); + stream.on("error", () => res.destroy()); + res.on("close", () => { + clearInterval(timer); + stream.destroy(); + }); + stream.pipe(res); + return; + } + if (route !== base + "/rpc" || req.method !== "POST") + throw failure("not_found", "Route not found."); + let body = ""; + for await (const chunk of req) { + body += chunk.toString(); + if (Buffer.byteLength(body) > 128000) + throw failure("invalid_request", "Request exceeds 128 KB."); + } + const request = Request.parse(JSON.parse(body)); + if (request.method === "shutdown") { + if (!admin) + throw failure("permission_denied", "Host authority required."); + if (request.generation !== controller.generation) + throw failure("generation_mismatch", "Lifecycle mismatch."); + send(200, { + ok: true, + protocol: 1, + generation: controller.generation, + result: { stopping: true }, + }); + setImmediate(() => void close()); + return; + } + const priority = [ + "stop", + "takeover", + "release", + "rebind", + "revoke", + ].includes(request.method); + if (inFlight >= 32 && !priority) + throw failure("overloaded", "Request limit reached."); inFlight++; counted = true; - const windowId = url.searchParams.get("windowId"); - const target = Target.parse( - windowId === null - ? { kind: "display" } - : { kind: "window", id: Number(windowId) }, - ); - const frame = await controller.observe(grant, target); - authenticate(req.headers.authorization, "viewer-read"); - send(200, { - ...frame, - recording: controller.recorder - .list() - .filter((r) => r.state === "recording" || r.state === "paused") - .map(({ id, state }) => ({ id, state })), + res.once("close", () => { + if (!res.writableEnded && request.method === "input") + void controller.cancelGrantInput(grant.id).catch(() => {}); }); - return; - } - if (route.startsWith(base + "/artifacts/") && req.method === "GET") { - authenticate(req.headers.authorization, "recording-read"); - const id = route.slice((base + "/artifacts/").length); - if (!/^[a-f0-9-]{36}$/.test(id)) - throw failure("not_found", "Artifact not found."); - const file = controller.recorder.path(id), - info = await stat(file); - let start = 0, - end = info.size - 1; - if (req.headers.range) { - const match = /^bytes=(\d+)-(\d*)$/.exec(req.headers.range); - if (!match) - throw failure("invalid_range", "Use one explicit byte range."); - start = +match[1]; - end = match[2] ? +match[2] : end; - if (start > end || end >= info.size) { - res.writeHead(416, { "Content-Range": `bytes */${info.size}` }); - res.end(); - return; - } + const result = await controller.call(request, grant, admin); + if ( + request.method !== "rebind" && + request.generation !== controller.generation + ) + throw failure( + "generation_mismatch", + "Lifecycle changed during the operation.", + [ + "health", + "state", + "apps", + "windows", + "observe", + "presence.list", + "recording.list", + ].includes(request.method) + ? "notDispatched" + : "unknown", + ); + if (request.method === "rebind") { + descriptor.generation = controller.generation; + await writeFile(path, JSON.stringify(descriptor), { mode: 0o600 }); } - res.writeHead(req.headers.range ? 206 : 200, { - "Content-Type": "video/mp4", - "Accept-Ranges": "bytes", - "Content-Length": end - start + 1, - "Content-Disposition": `attachment; filename="${id}.mp4"`, - ...(req.headers.range - ? { "Content-Range": `bytes ${start}-${end}/${info.size}` } - : {}), - }); - const stream = createReadStream(file, { start, end }); - const timer = setInterval(() => { - try { - authenticate(req.headers.authorization, "recording-read"); - } catch { - stream.destroy(); - res.destroy(); - } - }, 100); - stream.on("error", () => res.destroy()); - res.on("close", () => { - clearInterval(timer); - stream.destroy(); + send(200, { + ok: true, + protocol: 1, + generation: controller.generation, + result, }); - stream.pipe(res); - return; - } - if (route !== base + "/rpc" || req.method !== "POST") - throw failure("not_found", "Route not found."); - let body = ""; - for await (const chunk of req) { - body += chunk.toString(); - if (Buffer.byteLength(body) > 128000) - throw failure("invalid_request", "Request exceeds 128 KB."); - } - const request = Request.parse(JSON.parse(body)); - if (request.method === "shutdown") { - if (!admin) - throw failure("permission_denied", "Host authority required."); - if (request.generation !== controller.generation) - throw failure("generation_mismatch", "Lifecycle mismatch."); - send(200, { ok: true, result: { stopping: true } }); - setImmediate(() => void close()); - return; - } - const priority = [ - "stop", - "takeover", - "release", - "rebind", - "revoke", - ].includes(request.method); - if (inFlight >= 32 && !priority) - throw failure("overloaded", "Request limit reached."); - inFlight++; - counted = true; - res.once("close", () => { - if (!res.writableEnded && request.method === "input") - void controller.cancelGrantInput(grant.id).catch(() => {}); - }); - const result = await controller.call(request, grant, admin); - if ( - request.method !== "rebind" && - request.generation !== controller.generation - ) - throw failure( - "generation_mismatch", - "Lifecycle changed during the operation.", - ); - if (request.method === "rebind") { - descriptor.generation = controller.generation; - await writeFile(path, JSON.stringify(descriptor), { mode: 0o600 }); + } catch (e) { + if (!res.headersSent) + send( + (e as { code?: string }).code === "permission_denied" ? 403 : 400, + { ok: false, error: errorJSON(e) }, + ); + else res.destroy(); + } finally { + if (res.writableFinished || res.destroyed) finished(); } - send(200, { - ok: true, - protocol: 1, - generation: controller.generation, - result, - }); - } catch (e) { - if (!res.headersSent) - send( - (e as { code?: string }).code === "permission_denied" ? 403 : 400, - { ok: false, error: errorJSON(e) }, - ); - else res.destroy(); - } finally { - if (res.writableFinished || res.destroyed) finished(); - } - }); - server.maxConnections = 64; - server.maxRequestsPerSocket = 100; - server.requestTimeout = 15000; - server.headersTimeout = 10000; - await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); - const address = server.address(); - if (!address || typeof address === "string") throw Error("No listener"); - const origin = `http://127.0.0.1:${address.port}`; - const descriptor = { - protocol: 1, - display, - generation: controller.generation, - endpoint: origin + base, - token: master, - pid: process.pid, - }; - await mkdir(dirname(path), { recursive: true, mode: 0o700 }); - const dirInfo = await lstat(dirname(path)); - if ( - !dirInfo.isDirectory() || - dirInfo.isSymbolicLink() || - dirInfo.uid !== process.getuid?.() || - dirInfo.mode & 0o077 - ) - throw failure("permission_denied", "Unsafe broker directory."); - await writeFile(path, JSON.stringify(descriptor), { mode: 0o600 }); - await chmod(path, 0o600); - const close = async () => { - if (closed) return; - closed = true; - await controller.close(); - server.closeAllConnections(); - await new Promise((resolve) => server.close(() => resolve())); - await unlink(path).catch(() => {}); + }); + startupServer = server; + server.maxConnections = 64; + + server.requestTimeout = 15000; + server.headersTimeout = 10000; + await new Promise((resolve, reject) => { + server.once("error", reject); + server.listen(0, "127.0.0.1", resolve); + }); + const address = server.address(); + if (!address || typeof address === "string") throw Error("No listener"); + const origin = `http://127.0.0.1:${address.port}`; + const descriptor = { + protocol: 1, + display, + generation: controller.generation, + endpoint: origin + base, + token: master, + pid: process.pid, + }; + await mkdir(dirname(path), { recursive: true, mode: 0o700 }); + const dirInfo = await lstat(dirname(path)); + if ( + !dirInfo.isDirectory() || + dirInfo.isSymbolicLink() || + dirInfo.uid !== process.getuid?.() || + dirInfo.mode & 0o077 + ) + throw failure("permission_denied", "Unsafe broker directory."); + await writeFile(path, JSON.stringify(descriptor), { mode: 0o600 }); + await chmod(path, 0o600); + const close = async () => { + if (closed) return; + closed = true; + await controller.close(); + server.closeAllConnections(); + await new Promise((resolve) => server.close(() => resolve())); + // A lost Mac helper connection can release flock before HTTP teardown. + // Never unlink its descriptor here: a successor may already have replaced + // it. A stale private descriptor is probed on attach and replaced by the + // next owner, exactly as after an abrupt process exit. + if (!mac) await unlink(path).catch(() => {}); + macDriver?.close?.(); + lock.close(); + }; + return { controller, endpoint: descriptor.endpoint, path, close }; + } catch (error) { + await startupController?.close().catch(() => {}); + startupServer?.closeAllConnections(); + if (startupServer?.listening) + await new Promise((resolve) => + startupServer!.close(() => resolve()), + ); + startupDriver?.close?.(); lock.close(); - }; - return { controller, endpoint: descriptor.endpoint, path, close }; + throw error; + } } diff --git a/src/desktop/view.ts b/src/desktop/view.ts index b738a3f..c7a5f9c 100644 --- a/src/desktop/view.ts +++ b/src/desktop/view.ts @@ -1,25 +1,34 @@ export function desktopHTML(base: string) { - return `Opcode desktop
OpcodeConnecting…
Desktop stream`; } diff --git a/tests/browser-attach.test.ts b/tests/browser-attach.test.ts index da5d93c..4068d7a 100644 --- a/tests/browser-attach.test.ts +++ b/tests/browser-attach.test.ts @@ -1,14 +1,25 @@ -import { test } from 'node:test'; -import assert from 'node:assert/strict'; -import { BrowserTools } from '../src/browser/tools.js'; +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { BrowserTools } from "../src/browser/tools.js"; -test('attachment requires a local endpoint',async()=>{ - const tool=new BrowserTools(); - await assert.rejects(tool.call({operation:'attach',endpoint:'https://example.com'}),/loopback/); -}); -test('existing-browser integration through the shipped Node runtime', { skip: process.platform !== 'linux', timeout: 30000 }, async()=>{ - const { execFile } = await import('node:child_process'); - const { promisify } = await import('node:util'); - const result = await promisify(execFile)('node', ['scripts/test-browser-attach.mjs'], { timeout: 25000 }); - assert.match(result.stdout, /PASS existing-browser/); +test("attachment requires a local endpoint", async () => { + const tool = new BrowserTools(); + await assert.rejects( + tool.call({ operation: "attach", endpoint: "https://example.com" }), + /loopback/, + ); }); +(process.platform === "linux" ? test : test.skip)( + "existing-browser integration through the shipped Node runtime", + { timeout: 30000 }, + async () => { + const { execFile } = await import("node:child_process"); + const { promisify } = await import("node:util"); + const result = await promisify(execFile)( + "node", + ["scripts/test-browser-attach.mjs"], + { timeout: 25000 }, + ); + assert.match(result.stdout, /PASS existing-browser/); + }, +); diff --git a/tests/browser.test.ts b/tests/browser.test.ts index 70d1179..ff00226 100644 --- a/tests/browser.test.ts +++ b/tests/browser.test.ts @@ -1,32 +1,67 @@ -import { test } from 'node:test'; -import assert from 'node:assert/strict'; -import { createServer } from 'node:http'; -import { existsSync } from 'node:fs'; -import { BrowserDriver } from '../src/main/browser.js'; +import assert from "node:assert/strict"; +import { existsSync } from "node:fs"; +import { createServer } from "node:http"; +import { test } from "node:test"; +import { BrowserDriver } from "../src/main/browser.js"; -const chrome = process.env.TEST_CHROME_PATH ?? '/usr/bin/google-chrome'; -test('DOM route fills a real form, observes its result, and rejects stale refs', { skip: !existsSync(chrome) }, async () => { - const server = createServer((_req, res) => { - res.setHeader('Content-Type', 'text/html'); - res.end('Driver fixture

Waiting

'); - }); - await new Promise(resolve => server.listen(0, '127.0.0.1', resolve)); - const address = server.address(); - assert.ok(address && typeof address !== 'string'); - const driver = new BrowserDriver({ executablePath: chrome, headless: true, args: ['--no-sandbox'] }); - try { - let s = await driver.snapshot(); - await driver.execute({ kind: 'navigate', text: `http://127.0.0.1:${address.port}` }, s.id); - s = await driver.snapshot(); - const input = s.nodes.find(n => n.name === 'Name'); assert.ok(input); - await driver.execute({ kind: 'setValue', ref: input.ref, text: 'Jev test' }, s.id); - await assert.rejects(driver.execute({ kind: 'setValue', ref: input.ref, text: 'duplicate' }, s.id), /Stale/); - s = await driver.snapshot(); - const save = s.nodes.find(n => n.name === 'Save'); assert.ok(save); - await driver.execute({ kind: 'press', ref: save.ref }, s.id); - s = await driver.snapshot(); - assert.ok(s.nodes.some(n => n.name === 'Saved: Jev test')); - const abort = new AbortController(); abort.abort(); - await assert.rejects(driver.execute({ kind: 'navigate', text: 'https://example.com' }, s.id, abort.signal)); - } finally { await driver.close(); await new Promise(resolve => server.close(() => resolve())); } -}); +const chrome = process.env.TEST_CHROME_PATH ?? "/usr/bin/google-chrome"; +(existsSync(chrome) ? test : test.skip)( + "DOM route fills a real form, observes its result, and rejects stale refs", + async () => { + const server = createServer((_req, res) => { + res.setHeader("Content-Type", "text/html"); + res.end( + 'Driver fixture

Waiting

', + ); + }); + await new Promise((resolve) => + server.listen(0, "127.0.0.1", resolve), + ); + const address = server.address(); + assert.ok(address && typeof address !== "string"); + const driver = new BrowserDriver({ + executablePath: chrome, + headless: true, + args: ["--no-sandbox"], + }); + try { + let s = await driver.snapshot(); + await driver.execute( + { kind: "navigate", text: `http://127.0.0.1:${address.port}` }, + s.id, + ); + s = await driver.snapshot(); + const input = s.nodes.find((n) => n.name === "Name"); + assert.ok(input); + await driver.execute( + { kind: "setValue", ref: input.ref, text: "Jev test" }, + s.id, + ); + await assert.rejects( + driver.execute( + { kind: "setValue", ref: input.ref, text: "duplicate" }, + s.id, + ), + /Stale/, + ); + s = await driver.snapshot(); + const save = s.nodes.find((n) => n.name === "Save"); + assert.ok(save); + await driver.execute({ kind: "press", ref: save.ref }, s.id); + s = await driver.snapshot(); + assert.ok(s.nodes.some((n) => n.name === "Saved: Jev test")); + const abort = new AbortController(); + abort.abort(); + await assert.rejects( + driver.execute( + { kind: "navigate", text: "https://example.com" }, + s.id, + abort.signal, + ), + ); + } finally { + await driver.close(); + await new Promise((resolve) => server.close(() => resolve())); + } + }, +); diff --git a/tests/desktop-client.test.ts b/tests/desktop-client.test.ts new file mode 100644 index 0000000..2a77b19 --- /dev/null +++ b/tests/desktop-client.test.ts @@ -0,0 +1,126 @@ +import assert from "node:assert/strict"; +import { execFileSync, spawnSync } from "node:child_process"; +import { chmod, mkdtemp, readFile, rm, symlink } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { test } from "node:test"; +import { + desktopCall, + readCredential, + serviceURL, + writeCredential, +} from "../src/desktop/client.js"; + +const credential = { + endpoint: "https://mac.example.test/desktop", + token: "a".repeat(64), + generation: "boot-one", +}; +test("remote endpoints require TLS; loopback tunnels are allowed without URL credentials", () => { + for (const value of [ + credential.endpoint, + "http://127.0.0.1:4311", + "http://[::1]:4311/desktop", + ]) + assert.ok(serviceURL(value)); + for (const value of [ + "http://mac.example.test", + "http://100.64.0.1", + "https://user:secret@mac.example.test", + "https://mac.example.test?token=x", + "https://mac.example.test#secret", + "file:///tmp/socket", + "http://127.0.0.1.evil.test", + ]) + assert.throws(() => serviceURL(value), /HTTPS/); +}); +test("credentials are private, never overwritten, and symlinks are rejected", async () => { + const dir = await mkdtemp(join(tmpdir(), "cu-credentials-")); + try { + const path = join(dir, "user.json"); + await writeCredential(path, credential); + assert.deepEqual(await readCredential(path), credential); + await assert.rejects( + writeCredential(path, { ...credential, token: "b".repeat(64) }), + ); + assert.equal( + JSON.parse(await readFile(path, "utf8")).token, + credential.token, + ); + await symlink(path, join(dir, "link.json")); + await assert.rejects(readCredential(join(dir, "link.json"))); + await chmod(path, 0o644); + await assert.rejects(readCredential(path), /private/); + } finally { + await rm(dir, { recursive: true, force: true }); + } +}); +test("remote calls bind generation, disable redirects and never retry uncertain input", async () => { + let count = 0; + const transport = (async ( + url: string | URL | Request, + init?: RequestInit, + ) => { + count++; + assert.ok(init); + assert.equal(url, credential.endpoint + "/rpc"); + assert.equal(init?.redirect, "error"); + assert.equal( + (init.headers as Record).Authorization, + "Bearer " + credential.token, + ); + assert.equal(JSON.parse(init?.body as string).generation, "boot-one"); + throw new Error("socket lost"); + }) as typeof fetch; + await assert.rejects( + desktopCall(credential, "input", {}, { fetch: transport }), + (error: any) => + error.delivery === "unknown" && error.code === "connection_lost", + ); + assert.equal(count, 1); + const result = (async () => + Response.json({ + ok: true, + protocol: 1, + generation: "boot-two", + result: {}, + })) as typeof fetch; + await assert.rejects( + desktopCall(credential, "state", {}, { fetch: result }), + /lifecycle changed/, + ); + const wrong = (async () => Response.json(null)) as typeof fetch; + await assert.rejects( + desktopCall(credential, "input", {}, { fetch: wrong }), + (error: any) => error.delivery === "unknown", + ); +}); + +test("credential named pipes are rejected without waiting for a writer", async () => { + const dir = await mkdtemp(join(tmpdir(), "cu-credential-fifo-")); + try { + const path = join(dir, "credential.fifo"); + execFileSync("mkfifo", ["-m", "600", path]); + // Isolate the reader so a blocking-open regression cannot hang the suite. + const result = spawnSync( + process.execPath, + [ + "--eval", + ` + import { readCredential } from ${JSON.stringify(new URL("../src/desktop/client.ts", import.meta.url).href)}; + try { + await readCredential(${JSON.stringify(path)}); + process.exit(2); + } catch (error) { + process.exit(error.code === "permission_denied" ? 0 : 3); + } + `, + ], + { timeout: 3000, encoding: "utf8" }, + ); + assert.equal(result.error, undefined, result.error?.message); + assert.equal(result.status, 0, result.stderr); + } finally { + await rm(dir, { recursive: true, force: true }); + } +}); diff --git a/tests/desktop-controller.test.ts b/tests/desktop-controller.test.ts index 45ab3da..6031fc3 100644 --- a/tests/desktop-controller.test.ts +++ b/tests/desktop-controller.test.ts @@ -1,24 +1,26 @@ -import { test } from "node:test"; import assert from "node:assert/strict"; -import { mkdtemp, rm } from "node:fs/promises"; +import { randomUUID } from "node:crypto"; +import { mkdtemp, readFile, rm } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; -import { randomUUID } from "node:crypto"; -import { DesktopController } from "../src/desktop/controller.js"; -import { X11Backend } from "../src/desktop/backend.js"; +import { test } from "node:test"; import { Authority } from "../src/desktop/authority.js"; +import { X11Backend } from "../src/desktop/backend.js"; +import { DesktopController } from "../src/desktop/controller.js"; import { - scopes, - type Request, type Action, + type Request, + scopes, type Target, } from "../src/desktop/protocol.js"; + class Fixture extends X11Backend { delivered: Action[] = []; block = false; color = 0; width = 640; releaseCount = 0; + failRelease = false; override async geometry() { return { id: 1, x: 0, y: 0, width: this.width, height: 480 }; } @@ -35,6 +37,7 @@ class Fixture extends X11Backend { } override async input(_t: Target, a: Action, s: AbortSignal) { if (a.kind === "keyUp") { + if (this.failRelease) throw Error("driver disconnected"); this.releaseCount++; return; } @@ -82,7 +85,11 @@ test("one controller, takeover cancels queued input and releases held keys; fres }), a, ); - const failure = assert.rejects(action, /may have been delivered/); + const failure = assert.rejects(action, (error: unknown) => { + assert.equal((error as { delivery: string }).delivery, "unknown"); + assert.equal((error as { code: string }).code, "outcome_unknown"); + return true; + }); await new Promise((r) => setTimeout(r, 10)); const queued = c.call( request("input", { @@ -152,3 +159,233 @@ test("duplicate mutations do not replay; changed pixels, geometry and generation await rm(directory, { recursive: true, force: true }); } }); + +test("multiplayer cursors require presence scope and do not grant OS input; participant leave fences held input", async () => { + const directory = await mkdtemp(join(tmpdir(), "opcode-multiplayer-")); + const backend = new Fixture("", {}); + const c = new DesktopController(backend, "one", directory); + await c.init(); + try { + const viewer = c.authority.issue("viewer", ["viewer-read"], 60000); + const participant = c.authority.issue( + "alice", + ["viewer-read", "presence"], + 60000, + ); + const controller = c.authority.issue("bob", [...scopes], 60000); + await assert.rejects(c.call(request("presence.join"), viewer), /scope/); + const alice = (await c.call( + request("presence.join", { name: "Alice", role: "human" }), + participant, + )) as { id: string }; + await assert.rejects( + c.call(request("takeover", { participantId: alice.id }), participant), + /scope/, + ); + await assert.rejects( + c.call(request("takeover", { participantId: alice.id }), controller), + /another credential/, + ); + const bob = (await c.call( + request("presence.join", { name: "Bob" }), + controller, + )) as { id: string }; + const lease = (await c.call( + request("takeover", { participantId: bob.id }), + controller, + )) as { id: string }; + const observation = await c.observe(controller, { kind: "display" }); + await c.call( + request("input", { + leaseId: lease.id, + observationId: observation.observationId, + action: { kind: "keyDown", key: "Shift" }, + }), + controller, + ); + await c.call( + request("presence.leave", { participantId: alice.id }), + participant, + ); + assert.equal(c.multiplayerState().controller?.participantId, bob.id); + await c.call( + request("presence.leave", { participantId: bob.id }), + controller, + ); + assert.equal(c.multiplayerState().controller, null); + assert.equal(backend.releaseCount, 1); + await assert.rejects( + c.call(request("renew", { leaseId: lease.id }), controller), + /Acquire/, + ); + c.authority.revoke(controller.id); + await assert.rejects( + c.call(request("presence.join"), controller), + /revoked/, + ); + } finally { + await c.close(); + await rm(directory, { recursive: true, force: true }); + } +}); + +test("revocation and rebind remove multiplayer identities and invalidate participant-bound leases", async () => { + const directory = await mkdtemp(join(tmpdir(), "opcode-multiplayer-")); + const c = new DesktopController(new Fixture("", {}), "one", directory); + await c.init(); + try { + const host = c.authority.issue("host", [...scopes], 60000); + const guest = c.authority.issue("guest", [...scopes], 60000); + const participant = (await c.call(request("presence.join"), guest)) as { + id: string; + }; + await c.call(request("takeover", { participantId: participant.id }), guest); + await c.call(request("revoke", { id: guest.id }), host, true); + assert.deepEqual(c.multiplayerState(), { + participants: [], + controller: null, + }); + await c.call(request("presence.join"), host); + await c.call(request("rebind", { generation: "two" }), host, true); + assert.deepEqual(c.multiplayerState(), { + participants: [], + controller: null, + }); + await assert.rejects(c.call(request("presence.join"), host), /rebind/); + } finally { + await c.close(); + await rm(directory, { recursive: true, force: true }); + } +}); + +test("a busy observer cannot evict another participant's actionable observation", async () => { + const directory = await mkdtemp(join(tmpdir(), "opcode-multiplayer-")); + const backend = new Fixture("", {}); + const c = new DesktopController(backend, "one", directory); + await c.init(); + try { + const agent = c.authority.issue("agent", [...scopes], 60000); + const viewer = c.authority.issue("viewer", ["viewer-read"], 60000); + const lease = (await c.call(request("acquire"), agent)) as { id: string }; + const observation = await c.observe(agent, { kind: "display" }); + for (let i = 0; i < 40; i++) await c.observe(viewer, { kind: "display" }); + await c.call( + request("input", { + leaseId: lease.id, + observationId: observation.observationId, + action: { kind: "click", x: 1, y: 1 }, + }), + agent, + ); + assert.equal(backend.delivered.length, 1); + } finally { + await c.close(); + await rm(directory, { recursive: true, force: true }); + } +}); + +test("failed key release remains journaled and blocks acquisition until restart cleanup succeeds", async () => { + const directory = await mkdtemp(join(tmpdir(), "opcode-recovery-")); + const backend = new Fixture("", {}); + const controller = new DesktopController(backend, "one", directory); + await controller.init(); + try { + const grant = controller.authority.issue("agent", [...scopes], 60000); + const lease = (await controller.call(request("acquire"), grant)) as { + id: string; + }; + const observation = await controller.observe(grant, { kind: "display" }); + await controller.call( + request("input", { + leaseId: lease.id, + observationId: observation.observationId, + action: { kind: "keyDown", key: "Shift" }, + }), + grant, + ); + backend.failRelease = true; + await assert.rejects( + controller.call(request("stop"), grant), + /could not be released/, + ); + assert.deepEqual( + JSON.parse(await readFile(join(directory, "held-input.json"), "utf8")) + .keys, + ["Shift"], + ); + await assert.rejects( + controller.call(request("takeover"), grant), + /transition/, + ); + await controller.close(); + assert.deepEqual( + JSON.parse(await readFile(join(directory, "held-input.json"), "utf8")) + .keys, + ["Shift"], + ); + const recoveredBackend = new Fixture("", {}); + const recovered = new DesktopController(recoveredBackend, "two", directory); + try { + await recovered.init(); + assert.equal(recoveredBackend.releaseCount, 1); + assert.deepEqual( + JSON.parse(await readFile(join(directory, "held-input.json"), "utf8")) + .keys, + [], + ); + } finally { + await recovered.close(); + } + } finally { + await controller.close(); + await rm(directory, { recursive: true, force: true }); + } +}); + +test("transient shortcuts and clicks preserve preexisting held input until explicit release or takeover", async () => { + const directory = await mkdtemp(join(tmpdir(), "opcode-held-input-")); + const backend = new Fixture("", {}); + const c = new DesktopController(backend, "one", directory); + await c.init(); + try { + const grant = c.authority.issue("agent", [...scopes], 60000); + const human = c.authority.issue("human", [...scopes], 60000); + const lease = (await c.call(request("acquire"), grant)) as { id: string }; + const input = async (action: Action) => { + const observation = await c.observe(grant, { kind: "display" }); + await c.call( + request("input", { + leaseId: lease.id, + observationId: observation.observationId, + action, + }), + grant, + ); + }; + const held = async () => + JSON.parse(await readFile(join(directory, "held-input.json"), "utf8")); + await input({ kind: "keyDown", key: "Control" }); + await input({ kind: "buttonDown", button: 1 }); + await input({ kind: "key", key: "Control+a" }); + await input({ kind: "text", text: "hello", pasteKey: "Control+v" }); + await input({ kind: "click", x: 1, y: 1 }); + assert.deepEqual(await held(), { keys: ["Control"], buttons: [1] }); + await input({ kind: "keyUp", key: "Control" }); + await input({ kind: "buttonUp", button: 1 }); + assert.deepEqual(await held(), { keys: [], buttons: [] }); + await input({ kind: "keyDown", key: "Control" }); + await input({ kind: "buttonDown", button: 1 }); + await input({ kind: "key", key: "Control+a" }); + await input({ kind: "click", x: 1, y: 1 }); + await c.call(request("takeover"), human); + assert.equal(backend.releaseCount, 2); + assert.equal( + backend.delivered.filter((action) => action.kind === "buttonUp").length, + 2, + ); + assert.deepEqual(await held(), { keys: [], buttons: [] }); + } finally { + await c.close(); + await rm(directory, { recursive: true, force: true }); + } +}); diff --git a/tests/desktop-http.test.ts b/tests/desktop-http.test.ts new file mode 100644 index 0000000..df8463d --- /dev/null +++ b/tests/desktop-http.test.ts @@ -0,0 +1,158 @@ +import assert from "node:assert/strict"; +import { randomUUID } from "node:crypto"; +import { chmod, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { test } from "node:test"; +import { desktopCall } from "../src/desktop/client.js"; +import { startDesktop } from "../src/desktop/server.js"; + +// Exercise the real authenticated HTTP server without requiring an X server. +(process.platform === "linux" ? test : test.skip)( + "HTTP multiplayer isolates grants, publishes cursor state, and fences revoke/rebind", + async () => { + const directory = await mkdtemp(join(tmpdir(), "opcode-http-")); + const helper = join(directory, "helper.cjs"); + await writeFile( + helper, + `#!/usr/bin/env node +const op=process.argv[2]; +if(op==='display') console.log(JSON.stringify({id:1,x:0,y:0,width:2,height:2})); +else if(op==='state') console.log(JSON.stringify({focus:1,x:0,y:0})); +else if(op==='capture') process.stdout.write(Buffer.concat([Buffer.from('P6\\n2 2\\n255\\n'),Buffer.alloc(12)])); +else if(op==='list') console.log('[]'); +else console.log('{}'); +`, + ); + await chmod(helper, 0o700); + let service: Awaited> | undefined; + try { + service = await startDesktop({ + display: `:${100000 + Math.floor(Math.random() * 100000000)}`, + uid: process.getuid!(), + generation: "one", + origin: "http://127.0.0.1:4311", + helper, + directory: join(directory, "data"), + }); + const endpoint = service.endpoint; + const host = JSON.parse(await readFile(service.path, "utf8")) + .token as string; + const rpc = async ( + token: string, + method: string, + args: Record = {}, + generation = "one", + ) => { + const response = await fetch(endpoint + "/rpc", { + method: "POST", + headers: { + Authorization: `Bearer ${token}`, + "Content-Type": "application/json", + }, + body: JSON.stringify({ id: randomUUID(), generation, method, args }), + }); + return { + status: response.status, + ...((await response.json()) as { + ok: boolean; + result: any; + error?: { code: string }; + }), + }; + }; + const alice = ( + await rpc(host, "grant", { + subject: "alice", + scopes: ["viewer-read", "presence"], + ttlMs: 60000, + }) + ).result; + const bob = ( + await rpc(host, "grant", { + subject: "bob", + scopes: ["viewer-read", "presence", "input-control"], + ttlMs: 60000, + }) + ).result; + for (const [origin, expected] of [ + ["http://127.0.0.1:4311", 200], + [new URL(endpoint).origin, 200], + ["https://untrusted.example", 403], + ] as const) { + const response = await fetch(endpoint + "/session", { + headers: { Authorization: `Bearer ${alice.token}`, Origin: origin }, + }); + assert.equal(response.status, expected); + } + const anonymous = await fetch(endpoint + "/frame"); + assert.equal(anonymous.status, 403); + const ap = (await rpc(alice.token, "presence.join", { name: "Alice" })) + .result; + const bp = (await rpc(bob.token, "presence.join", { name: "Bob" })) + .result; + assert.equal( + (await rpc(alice.token, "takeover", { participantId: ap.id })).status, + 403, + ); + assert.equal( + ( + await rpc(bob.token, "presence.update", { + participantId: ap.id, + cursor: { x: 0.2, y: 0.3 }, + }) + ).ok, + false, + ); + assert.equal( + ( + await rpc(alice.token, "presence.update", { + participantId: ap.id, + cursor: { x: 0.2, y: 0.3 }, + }) + ).ok, + true, + ); + const lease = (await rpc(bob.token, "takeover", { participantId: bp.id })) + .result; + assert.ok(lease.id); + const frameResponse = await fetch(endpoint + "/frame", { + headers: { Authorization: `Bearer ${alice.token}` }, + }); + assert.equal(frameResponse.status, 200); + const frame = (await frameResponse.json()) as any; + assert.equal(frame.controller.participantId, bp.id); + assert.deepEqual( + frame.participants.find((p: any) => p.id === ap.id).cursor, + { x: 0.2, y: 0.3 }, + ); + assert.ok(frame.image.base64); + assert.equal( + (await rpc(alice.token, "state")).result.lease.subject, + "bob", + ); + await rpc(host, "revoke", { id: bob.id }); + const state = (await rpc(alice.token, "presence.list")).result; + assert.equal(state.controller, null); + assert.equal(state.participants.length, 1); + assert.equal((await rpc(bob.token, "state")).status, 403); + await rpc(host, "rebind", { generation: "two" }); + assert.equal((await rpc(alice.token, "state", {}, "two")).status, 403); + assert.deepEqual((await rpc(host, "presence.list", {}, "two")).result, { + participants: [], + controller: null, + }); + assert.deepEqual( + await desktopCall( + { endpoint, token: host, generation: "two" }, + "shutdown", + {}, + ), + { stopping: true }, + ); + } finally { + await service?.close(); + await rm(directory, { recursive: true, force: true }); + } + }, +); diff --git a/tests/desktop-platform.test.ts b/tests/desktop-platform.test.ts new file mode 100644 index 0000000..90d2085 --- /dev/null +++ b/tests/desktop-platform.test.ts @@ -0,0 +1,36 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { + canonical, + displayKey, + registryPath, +} from "../src/desktop/registry.js"; +import { Config, startDesktop } from "../src/desktop/server.js"; +import { registryPath as x11Path } from "../src/linux/display-registry.js"; + +test("Mac registry is stable and separate from X11 display identities", () => { + assert.equal(canonical("macos"), "macos"); + assert.equal(canonical("unix:01"), ":1.0"); + assert.equal(registryPath(":1"), x11Path(":1")); + assert.equal(registryPath("macos"), registryPath("macos")); + assert.notEqual(displayKey("macos"), displayKey(":0")); + assert.throws(() => canonical("remote:0")); +}); +test("Mac configuration needs no X11 helper and rejects arbitrary displays", () => { + assert.equal( + Config.parse({ display: "macos", uid: 501, generation: "test" }).helper, + "", + ); + assert.equal( + Config.safeParse({ display: "evil-host:0", uid: 501, generation: "test" }) + .success, + false, + ); +}); +test("Unsupported platform fails before allocating a broker", async () => { + const display = process.platform === "darwin" ? ":99" : "macos"; + await assert.rejects( + startDesktop({ display, uid: process.getuid?.() ?? 0, generation: "test" }), + (error: { code?: string }) => error.code === "unsupported", + ); +}); diff --git a/tests/desktop-presence.test.ts b/tests/desktop-presence.test.ts new file mode 100644 index 0000000..fc4a6a9 --- /dev/null +++ b/tests/desktop-presence.test.ts @@ -0,0 +1,57 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { Authority } from "../src/desktop/authority.js"; +import { Presence } from "../src/desktop/presence.js"; +import { methodArguments } from "../src/desktop/protocol.js"; + +test("participants are isolated by grant, bounded, and use visual-only normalized cursors", () => { + let now = 1000; + const authority = new Authority("one", () => now); + const presence = new Presence(authority, () => now); + const a = authority.issue("alice", ["presence"], 60000); + const b = authority.issue("bob", ["presence"], 60000); + const first = presence.join(a, "Alice", "human"); + const second = presence.join(a, "Other tab", "agent"); + assert.notEqual(first.id, second.id); + assert.throws( + () => presence.update(b, first.id, { x: 0.5, y: 0.5 }), + /another credential/, + ); + assert.throws(() => presence.leave(b, first.id), /another credential/); + assert.throws(() => + methodArguments["presence.update"].parse({ + participantId: first.id, + cursor: { x: 2, y: 0 }, + }), + ); + const update = presence.update(a, first.id, { x: 0.5, y: 0.25 }); + assert.deepEqual(update.cursor, { x: 0.5, y: 0.25 }); + assert.throws(() => presence.update(a, first.id, null), /limited/); + now += 100; + assert.deepEqual( + presence.update(a, first.id, undefined).cursor, + update.cursor, + ); + for (let i = 0; i < 6; i++) presence.join(a, `tab${i}`, "human"); + assert.throws(() => presence.join(a, "overflow", "human"), /limit/); + presence.leave(a, first.id); + assert.equal(presence.list().length, 7); +}); + +test("presence expires after disconnect and is removed on grant revocation and generation changes", () => { + let now = 1000; + const authority = new Authority("one", () => now); + const presence = new Presence(authority, () => now); + const grant = authority.issue("alice", ["presence"], 60000); + const p = presence.join(grant, undefined, "human"); + now += 15001; + assert.equal(presence.has(p.id), false); + assert.throws(() => presence.update(grant, p.id, null), /expired/); + presence.join(grant, undefined, "human"); + authority.revoke(grant.id); + assert.deepEqual(presence.list(), []); + const next = authority.issue("bob", ["presence"], 1000); + assert.equal(presence.join(next, undefined, "human").expiresAt, now + 1000); + authority.rebind("two"); + assert.deepEqual(presence.list(), []); +}); diff --git a/tests/desktop-remote.test.ts b/tests/desktop-remote.test.ts new file mode 100644 index 0000000..1a1b38f --- /dev/null +++ b/tests/desktop-remote.test.ts @@ -0,0 +1,298 @@ +import assert from "node:assert/strict"; +import { execFile } from "node:child_process"; +import { once } from "node:events"; +import { + chmod, + mkdtemp, + readFile, + rm, + stat, + writeFile, +} from "node:fs/promises"; +import { createServer as httpServer } from "node:http"; +import { createServer as netServer } from "node:net"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { test } from "node:test"; +import { promisify } from "node:util"; +import { readCredential } from "../src/desktop/client.js"; +import { + openSSHTunnel, + remoteCredentialCommand, + sshTarget, +} from "../src/desktop/remote.js"; + +const exec = promisify(execFile); + +const fakeSSH = `#!/usr/bin/env node +const net = require('node:net'); +const fs = require('node:fs'); +const args = process.argv.slice(2); +fs.appendFileSync(process.env.CU_TEST_SSH_LOG, JSON.stringify(args)+'\\n'); +if (!args.includes('-N')) { process.stdout.write(process.env.CU_TEST_REMOTE_CREDENTIAL); process.exit(0); } +const spec = args[args.indexOf('-L') + 1].split(':'); +const server = net.createServer(socket => { + const remote = net.connect(Number(spec[3]), spec[2]); + remote.on('error', () => socket.destroy()); socket.on('error', () => remote.destroy()); + socket.on('close', () => remote.destroy()); remote.on('close', () => socket.destroy()); + socket.pipe(remote); remote.pipe(socket); +}); +server.once('error', () => process.exit(1)); +server.listen(Number(spec[1]), spec[0], () => { + setTimeout(() => { + fs.writeFileSync(process.env.CU_TEST_SSH_MARKER, 'ready'); + process.stdout.write('OPCODE_TUNNEL_READY'); + }, 100); +}); +`; +async function freePort() { + const server = netServer(); + server.listen(0, "127.0.0.1"); + await once(server, "listening"); + const port = (server.address() as { port: number }).port; + await new Promise((resolve) => server.close(() => resolve())); + return port; +} +async function fixture(denied = false) { + const directory = await mkdtemp(join(tmpdir(), "cu-ssh-test-")); + const marker = join(directory, "ready"); + const log = join(directory, "ssh.log"); + await writeFile(join(directory, "ssh"), fakeSSH); + await chmod(join(directory, "ssh"), 0o700); + const token = "viewer-only-private-token"; + const requests: { method: string; authorization?: string; ready: boolean }[] = + []; + const server = httpServer(async (req, res) => { + let body = ""; + for await (const chunk of req) body += chunk; + const request = JSON.parse(body); + const ready = await stat(marker).then( + () => true, + () => false, + ); + requests.push({ + method: request.method, + authorization: req.headers.authorization, + ready, + }); + res.setHeader("Content-Type", "application/json"); + // This grant intentionally has no observe scope: readiness must use presence.list. + if ( + denied || + request.method !== "presence.list" || + req.headers.authorization !== `Bearer ${token}` + ) { + res.statusCode = 403; + res.end( + JSON.stringify({ + ok: false, + error: { code: "permission_denied", message: "Viewer scope only." }, + }), + ); + } else + res.end( + JSON.stringify({ + ok: true, + protocol: 1, + generation: "test-generation", + result: { participants: [] }, + }), + ); + }); + server.listen(0, "127.0.0.1"); + await once(server, "listening"); + const port = (server.address() as { port: number }).port; + const original = new Map(); + for (const [key, value] of Object.entries({ + PATH: directory + ":" + process.env.PATH, + CU_TEST_SSH_LOG: log, + CU_TEST_SSH_MARKER: marker, + CU_TEST_REMOTE_CREDENTIAL: JSON.stringify({ + endpoint: `http://127.0.0.1:${port}/desktop`, + token, + generation: "test-generation", + }), + })) { + original.set(key, process.env[key]); + process.env[key] = value; + } + return { + directory, + requests, + log, + token, + async close() { + for (const [key, value] of original) + if (value === undefined) delete process.env[key]; + else process.env[key] = value; + server.closeAllConnections(); + await new Promise((resolve) => server.close(() => resolve())); + await rm(directory, { recursive: true, force: true }); + }, + }; +} + +test("SSH target validation rejects option injection and shell metacharacters", () => { + for (const input of [ + "-oProxyCommand=evil", + "user@-host", + "host;touch", + "host\n", + "user@host:22", + "a b", + "$(evil)", + "", + ]) + assert.throws(() => sshTarget(input)); + for (const input of [ + "my-tailnet-mac", + "alice@mac.example.ts.net", + "my_alias", + ]) + assert.equal(sshTarget(input), input); +}); +test("Remote credential command treats malicious path content literally", async () => { + const directory = await mkdtemp(join(tmpdir(), "cu-path-test-")); + try { + const path = join( + directory, + "quote' $(printf exploited) `printf bad` ; $HOME.json", + ); + await writeFile(path, "literal-only"); + const { stdout } = await exec("/bin/sh", [ + "-c", + remoteCredentialCommand(path), + ]); + assert.equal(stdout, "literal-only"); + for (const input of [ + "relative.json", + "/tmp/new\nline", + "/tmp/null\0byte", + "/tmp/cr\rfile", + ]) + assert.throws(() => remoteCredentialCommand(input)); + } finally { + await rm(directory, { recursive: true, force: true }); + } +}); + +// Tests run sequentially because they temporarily select a fake ssh via PATH. +test("waits for forwarding readiness, supports viewer credentials, and deletes credentials on close", async () => { + const f = await fixture(); + let tunnel: Awaited> | undefined; + try { + const output = join(f.directory, "local.json"); + tunnel = await openSSHTunnel({ + host: "alice@mac", + remoteCredentialFile: "/Users/alice/viewer.json", + output, + port: await freePort(), + }); + assert.deepEqual(f.requests, [ + { + method: "presence.list", + authorization: `Bearer ${f.token}`, + ready: true, + }, + ]); + const credential = await readCredential(output); + assert.equal(credential.token, f.token); + assert.match(credential.endpoint, /^http:\/\/127\.0\.0\.1:\d+\/desktop$/); + const invocations = (await readFile(f.log, "utf8")) + .trim() + .split("\n") + .map((line) => JSON.parse(line) as string[]); + assert.equal(invocations.length, 2); + for (const args of invocations) { + assert.ok(args.includes("StrictHostKeyChecking=yes")); + assert.ok(args.includes("BatchMode=yes")); + assert.ok(!args.some((arg) => arg.includes(f.token))); + } + assert.ok(invocations[1]!.includes("ExitOnForwardFailure=yes")); + await tunnel.close(); + await assert.rejects( + readFile(output), + (error: NodeJS.ErrnoException) => error.code === "ENOENT", + ); + await tunnel.close(); + } finally { + await tunnel?.close(); + await f.close(); + } +}); +test("occupied local port receives no credential or readiness request", async () => { + const f = await fixture(); + let received = 0; + const occupied = httpServer((_req, res) => { + received++; + res.end("unrelated"); + }); + occupied.listen(0, "127.0.0.1"); + await once(occupied, "listening"); + try { + const output = join(f.directory, "must-not-exist.json"); + await assert.rejects( + openSSHTunnel({ + host: "mac", + remoteCredentialFile: "/private/viewer.json", + output, + port: (occupied.address() as { port: number }).port, + }), + /SSH closed before forwarding/, + ); + assert.equal(received, 0); + assert.equal(f.requests.length, 0); + await assert.rejects( + readFile(output), + (error: NodeJS.ErrnoException) => error.code === "ENOENT", + ); + } finally { + occupied.closeAllConnections(); + await new Promise((resolve) => occupied.close(() => resolve())); + await f.close(); + } +}); +test("authorization failure closes forwarding and leaves no credential", async () => { + const f = await fixture(true); + try { + const output = join(f.directory, "denied.json"); + const port = await freePort(); + await assert.rejects( + openSSHTunnel({ + host: "mac", + remoteCredentialFile: "/private/viewer.json", + output, + port, + }), + /Viewer scope only/, + ); + await assert.rejects( + readFile(output), + (error: NodeJS.ErrnoException) => error.code === "ENOENT", + ); + const probe = netServer(); + probe.listen(port, "127.0.0.1"); + await once(probe, "listening"); + await new Promise((resolve) => probe.close(() => resolve())); + } finally { + await f.close(); + } +}); +test("pre-existing output credentials are preserved on setup failure", async () => { + const f = await fixture(); + try { + const output = join(f.directory, "existing.json"); + await writeFile(output, "do not overwrite", { mode: 0o600 }); + await assert.rejects( + openSSHTunnel({ + host: "mac", + remoteCredentialFile: "/private/viewer.json", + output, + port: await freePort(), + }), + ); + assert.equal(await readFile(output, "utf8"), "do not overwrite"); + } finally { + await f.close(); + } +}); diff --git a/tests/desktop-view.test.ts b/tests/desktop-view.test.ts new file mode 100644 index 0000000..f93163e --- /dev/null +++ b/tests/desktop-view.test.ts @@ -0,0 +1,281 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; +import vm from "node:vm"; +import { desktopHTML } from "../src/desktop/view.js"; + +class Element { + textContent = ""; + value = ""; + hidden = false; + disabled = false; + className = ""; + style: Record = {}; + children: Element[] = []; + parent?: Element; + src?: string; + append(...nodes: Element[]) { + this.children.push(...nodes); + for (const node of nodes) node.parent = this; + } + replaceChildren(...nodes: Element[]) { + this.children = []; + this.append(...nodes); + } + get lastChild() { + const child = this.children.at(-1); + assert.ok(child); + return child; + } + remove() { + if (this.parent) + this.parent.children = this.parent.children.filter((n) => n !== this); + } + removeAttribute(key: string) { + if (key === "src") this.src = undefined; + } + focus() {} + blur() {} + setPointerCapture() {} + getBoundingClientRect() { + return { left: 0, top: 0, width: 100, height: 100 }; + } +} +function nextTimer(timers: Array<() => Promise | void>) { + const timer = timers.shift(); + assert.ok(timer); + return timer(); +} +const settle = () => new Promise((resolve) => setTimeout(resolve, 0)); +async function harness(admin = false) { + const elements = new Map(); + const element = (id: string) => { + if (!elements.has(id)) elements.set(id, new Element()); + const node = elements.get(id); + assert.ok(node); + return node; + }; + const calls: { method: string; args: Record }[] = []; + const timeouts: Array<() => Promise | void> = []; + const intervals: Array<() => Promise | void> = []; + let fail = false; + let controller: { participantId: string; subject?: string } | null = null; + const members: Array<{ + id: string; + name: string; + color: string; + cursor: { + x: number; + y: number; + target?: { kind: string; id?: number }; + } | null; + }> = []; + const document = { + onvisibilitychange: () => {}, + hidden: false, + getElementById: element, + createElement: () => new Element(), + }; + const listeners = new Map void>(); + const window = { + addEventListener: (name: string, callback: () => void) => + listeners.set(name, callback), + }; + const context = vm.createContext({ + AbortSignal, + document, + window, + location: { hash: "#token", pathname: "/view", search: "" }, + sessionStorage: { + getItem: () => null, + setItem: () => {}, + removeItem: () => {}, + }, + history: { replaceState: () => {} }, + crypto: { randomUUID: () => "request" }, + setTimeout: (f: () => Promise | void) => timeouts.push(f), + setInterval: (f: () => Promise | void) => intervals.push(f), + fetch: async (url: string, init: { body: string }) => { + if (fail) throw Error("Network lost"); + let data: unknown; + if (url.endsWith("/session")) + data = { + generation: "generation", + admin, + scopes: ["input-control", "viewer-read", "presence"], + }; + else if (url.endsWith("/frame")) + data = { + participants: members, + controller, + image: { width: 100, height: 100, base64: "png" }, + observationId: "observation", + }; + else { + const body = JSON.parse(init.body); + calls.push(body); + const response: { ok: boolean; result: unknown } = { + ok: true, + result: {}, + }; + data = response; + if (body.method === "presence.join") { + response.result = { + id: "self", + name: body.args.name, + color: "#fff", + cursor: null, + }; + members.push({ + id: "self", + name: body.args.name, + color: "#fff", + cursor: null, + }); + } + if (body.method === "takeover") { + response.result = { id: "lease" }; + controller = { participantId: "self" }; + } + if (body.method === "release") controller = null; + } + return { ok: true, json: async () => data }; + }, + }); + const match = desktopHTML("/api").match(/