From 1ead7684a739bc178069e32943e70baab7e4421f Mon Sep 17 00:00:00 2001 From: Swaraj Bachu Date: Sat, 3 Oct 2026 22:16:45 +0000 Subject: [PATCH 01/10] feat: add remote multiplayer desktop control for Mac and Linux --- native/macos/Driver.swift | 213 ++++++++++ src/desktop/backend.ts | 27 +- src/desktop/cli.ts | 336 +++++++++++++--- src/desktop/client.ts | 161 ++++++++ src/desktop/controller.ts | 197 +++++++--- src/desktop/index.ts | 20 +- src/desktop/mac-backend.ts | 112 ++++++ src/desktop/mac-driver.ts | 164 ++++++++ src/desktop/presence.ts | 104 +++++ src/desktop/protocol.ts | 34 +- src/desktop/registry.ts | 25 ++ src/desktop/remote.ts | 207 ++++++++++ src/desktop/server.ts | 640 +++++++++++++++++-------------- src/desktop/view.ts | 45 ++- tests/desktop-client.test.ts | 96 +++++ tests/desktop-controller.test.ts | 205 +++++++++- tests/desktop-http.test.ts | 142 +++++++ tests/desktop-platform.test.ts | 36 ++ tests/desktop-presence.test.ts | 57 +++ tests/desktop-remote.test.ts | 298 ++++++++++++++ tests/desktop-view.test.ts | 281 ++++++++++++++ tests/mac-backend.test.ts | 103 +++++ tests/mac-driver.test.ts | 112 ++++++ 23 files changed, 3183 insertions(+), 432 deletions(-) create mode 100644 src/desktop/client.ts create mode 100644 src/desktop/mac-backend.ts create mode 100644 src/desktop/mac-driver.ts create mode 100644 src/desktop/presence.ts create mode 100644 src/desktop/registry.ts create mode 100644 src/desktop/remote.ts create mode 100644 tests/desktop-client.test.ts create mode 100644 tests/desktop-http.test.ts create mode 100644 tests/desktop-platform.test.ts create mode 100644 tests/desktop-presence.test.ts create mode 100644 tests/desktop-remote.test.ts create mode 100644 tests/desktop-view.test.ts create mode 100644 tests/mac-backend.test.ts create mode 100644 tests/mac-driver.test.ts diff --git a/native/macos/Driver.swift b/native/macos/Driver.swift index 6cfcc15..29873d1 100644 --- a/native/macos/Driver.swift +++ b/native/macos/Driver.swift @@ -22,6 +22,7 @@ struct SavedElement { var visualCapturedAt: Date? var lastCaptureImage: CGImage? var virtualCursor: CGPoint? + var desktopHeldModifiers: CGEventFlags = [] let foregroundAllowed = ProcessInfo.processInfo.environment["JEV_INTERACTION_MODE"] == "foreground" var activatedRenderers = Set() var observationErrors = 0 @@ -592,8 +593,220 @@ struct SavedElement { return found.values.sorted { $0.name < $1.name } } + // Remote desktop operations stay in this permission-owning helper. The broker + // serializes physical input; participant cursors are overlays, not HID devices. + func desktopWindows() throws -> [[String: Any]] { + guard CGPreflightScreenCaptureAccess() else { throw DriverFailure(code: "permission_denied", message: "Grant Screen Recording to Opcode.") } + let entries = CGWindowListCopyWindowInfo([.optionOnScreenOnly, .excludeDesktopElements], kCGNullWindowID) as? [[String: Any]] ?? [] + return entries.compactMap { item in + guard let id = item[kCGWindowNumber as String] as? Int, + let pid = item[kCGWindowOwnerPID as String] as? Int, pid > 0, + let bounds = item[kCGWindowBounds as String] as? NSDictionary, + let rect = CGRect(dictionaryRepresentation: bounds), rect.width > 0, rect.height > 0, + (item[kCGWindowLayer as String] as? Int) == 0 else { return nil } + let frame = rect.integral + return ["id": id, "pid": pid, "title": item[kCGWindowName as String] as? String ?? "", + "x": frame.minX, "y": frame.minY, "width": Int(frame.width), "height": Int(frame.height)] + } + } + + func desktopGeometry(_ request: [String: Any]) throws -> [String: Any] { + guard let target = request["target"] as? [String: Any], let kind = target["kind"] as? String else { + throw DriverFailure(code: "InvalidRequest", message: "Desktop target required.") + } + if kind == "display" { + let id = CGMainDisplayID(), frame = CGDisplayBounds(id).integral + return ["id": Int(id), "x": frame.minX, "y": frame.minY, "width": Int(frame.width), "height": Int(frame.height)] + } + guard kind == "window", let id = target["id"] as? Int, + let window = try desktopWindows().first(where: { $0["id"] as? Int == id }) else { + throw DriverFailure(code: "stale_observation", message: "Window is no longer visible.") + } + return window + } + + func desktopState() -> [String: Any] { + let cursor = CGEvent(source: nil)?.location ?? .zero + var focus = 0 + if let app = NSWorkspace.shared.frontmostApplication { + let ax = AXUIElementCreateApplication(app.processIdentifier) + if let raw = value(ax, "AXFocusedWindow"), CFGetTypeID(raw) == AXUIElementGetTypeID() { + focus = Int(backgroundInput.windowID(raw as! AXUIElement) ?? 0) + } + } + return ["focus": focus, "x": cursor.x, "y": cursor.y] + } + + func desktopCapture(_ request: [String: Any]) async throws -> [String: Any] { + guard CGPreflightScreenCaptureAccess() else { throw DriverFailure(code: "permission_denied", message: "Grant Screen Recording to Opcode.") } + guard #available(macOS 14.0, *) else { throw DriverFailure(code: "unsupported", message: "Remote desktop requires macOS 14 or later.") } + let geometry = try desktopGeometry(request) + let width = geometry["width"] as! Int, height = geometry["height"] as! Int + guard width > 0, height > 0, width <= 8192, height <= 8192 else { + throw DriverFailure(code: "unsupported", message: "Desktop dimensions exceed the capture limit.") + } + let content = try await SCShareableContent.excludingDesktopWindows(false, onScreenWindowsOnly: true) + let filter: SCContentFilter + if (request["target"] as? [String: Any])?["kind"] as? String == "window" { + guard let window = content.windows.first(where: { Int($0.windowID) == geometry["id"] as? Int }) else { + throw DriverFailure(code: "stale_observation", message: "Window disappeared before capture.") + } + filter = SCContentFilter(desktopIndependentWindow: window) + } else { + guard let display = content.displays.first(where: { $0.displayID == CGMainDisplayID() }) else { + throw DriverFailure(code: "display_unavailable", message: "Primary display is unavailable.") + } + filter = SCContentFilter(display: display, excludingWindows: []) + } + let config = SCStreamConfiguration() + config.width = width; config.height = height; config.showsCursor = false + config.ignoreShadowsSingleWindow = true; config.ignoreGlobalClipSingleWindow = true + let image = try await SCScreenshotManager.captureImage(contentFilter: filter, configuration: config) + guard NSDictionary(dictionary: geometry).isEqual(to: try desktopGeometry(request)), + let png = NSBitmapImageRep(cgImage: image).representation(using: .png, properties: [:]), png.count <= 12_000_000 else { + throw DriverFailure(code: "stale_observation", message: "Display changed during capture or image exceeds limit.") + } + var sample = [UInt8](repeating: 0, count: 64 * 48 * 4) + let sampled = sample.withUnsafeMutableBytes { bytes -> Bool in + guard let context = CGContext(data: bytes.baseAddress, width: 64, height: 48, bitsPerComponent: 8, bytesPerRow: 64 * 4, + space: CGColorSpaceCreateDeviceRGB(), bitmapInfo: CGImageAlphaInfo.premultipliedLast.rawValue) else { return false } + context.draw(image, in: CGRect(x: 0, y: 0, width: 64, height: 48)); return true + } + guard sampled else { throw DriverFailure(code: "CaptureFailed", message: "Could not sample desktop frame.") } + return ["geometry": geometry, "base64": png.base64EncodedString(), "sample": Data(sample).base64EncodedString()] + } + + func desktopInput(_ request: [String: Any]) throws -> [String: Any] { + try requireAX() + guard request["foregroundApproved"] as? Bool == true, + let action = request["action"] as? [String: Any], let kind = action["kind"] as? String else { + throw DriverFailure(code: "permission_denied", message: "Desktop input requires explicit foreground approval.") + } + let releasing = kind == "keyUp" || kind == "buttonUp" + let geometry = try desktopGeometry(request) + if !releasing { + guard let expected = request["expectedGeometry"] as? [String: Any], NSDictionary(dictionary: expected).isEqual(to: geometry) else { + throw DriverFailure(code: "stale_observation", message: "Desktop geometry changed before input.") + } + if (request["target"] as? [String: Any])?["kind"] as? String == "window", kind != "focus" { + guard desktopState()["focus"] as? Int == geometry["id"] as? Int else { + throw DriverFailure(code: "stale_observation", message: "Focus the target window and observe again before physical input.") + } + } + if let app = NSWorkspace.shared.frontmostApplication { + let ax = AXUIElementCreateApplication(app.processIdentifier) + if let raw = value(ax, "AXFocusedUIElement"), CFGetTypeID(raw) == AXUIElementGetTypeID() { + let field = raw as! AXUIElement + guard string(field, "AXRole") != "AXSecureTextField", string(field, "AXSubrole") != "AXSecureTextField" else { + throw DriverFailure(code: "permission_denied", message: "Remote input is disabled in protected fields.") + } + } + } + } + let origin = CGPoint(x: geometry["x"] as? Double ?? 0, y: geometry["y"] as? Double ?? 0) + let width = geometry["width"] as! Int, height = geometry["height"] as! Int + func point(_ x: String, _ y: String) throws -> CGPoint { + guard let px = action[x] as? Double, let py = action[y] as? Double, + px.isFinite, py.isFinite, px >= 0, py >= 0, px < Double(width), py < Double(height) else { + throw DriverFailure(code: "invalid_coordinates", message: "Input must be inside the observed image.") + } + return CGPoint(x: origin.x + px, y: origin.y + py) + } + func mouse(_ type: CGEventType, _ p: CGPoint, _ button: CGMouseButton = .left, count: Int64 = 1) throws { + guard let event = CGEvent(mouseEventSource: nil, mouseType: type, mouseCursorPosition: p, mouseButton: button) else { + throw DriverFailure(code: "unsupported", message: "Could not allocate mouse event.", delivery: "unknown") + } + event.flags = desktopHeldModifiers + event.setIntegerValueField(.mouseEventClickState, value: count); event.post(tap: .cghidEventTap) + } + let codes: [String: CGKeyCode] = ["a":0,"s":1,"d":2,"f":3,"h":4,"g":5,"z":6,"x":7,"c":8,"v":9,"b":11,"q":12,"w":13,"e":14,"r":15,"y":16,"t":17,"1":18,"2":19,"3":20,"4":21,"6":22,"5":23,"9":25,"7":26,"8":28,"0":29,"o":31,"u":32,"i":34,"p":35,"l":37,"j":38,"k":40,"n":45,"m":46,"Enter":36,"Tab":48,"Space":49,"Backspace":51,"Escape":53,"Meta":55,"Shift":56,"Alt":58,"Control":59,"Home":115,"PageUp":116,"Delete":117,"End":119,"PageDown":121,"ArrowLeft":123,"ArrowRight":124,"ArrowDown":125,"ArrowUp":126,"F1":122,"F2":120,"F3":99,"F4":118,"F5":96,"F6":97,"F7":98,"F8":100,"F9":101,"F10":109,"F11":103,"F12":111] + func code(_ key: String) -> CGKeyCode? { codes[key] ?? (key.count == 1 ? codes[key.lowercased()] : nil) } + func keyboard(_ key: String, _ down: Bool, _ flags: CGEventFlags = []) throws { + guard let c = code(key), let event = CGEvent(keyboardEventSource: nil, virtualKey: c, keyDown: down) else { + throw DriverFailure(code: "unsupported", message: "Unsupported named key.") + } + event.flags = flags.union(desktopHeldModifiers); event.post(tap: .cghidEventTap) + } + switch kind { + case "click", "doubleClick", "rightClick", "hover", "drag": + let start = try point("x", "y") + let end = kind == "drag" ? try point("toX", "toY") : start + if kind == "hover" { try mouse(.mouseMoved, start) } + else if kind == "rightClick" { try mouse(.rightMouseDown, start, .right); try mouse(.rightMouseUp, start, .right) } + else { + let count = kind == "doubleClick" ? 2 : 1 + for index in 1...count { + try mouse(.leftMouseDown, start, count: Int64(index)) + if kind == "drag" { + for step in 1...12 { + let f = Double(step) / 12 + try mouse(.leftMouseDragged, CGPoint(x: start.x + (end.x - start.x) * f, y: start.y + (end.y - start.y) * f)) + Thread.sleep(forTimeInterval: 0.01) + } + } + try mouse(.leftMouseUp, end, count: Int64(index)) + } + } + case "buttonDown", "buttonUp": + guard let button = action["button"] as? Int, (1...3).contains(button) else { throw DriverFailure(code: "InvalidRequest", message: "Invalid button.") } + let p = CGEvent(source: nil)?.location ?? .zero + let down = kind == "buttonDown" + try mouse(button == 1 ? (down ? .leftMouseDown : .leftMouseUp) : button == 3 ? (down ? .rightMouseDown : .rightMouseUp) : (down ? .otherMouseDown : .otherMouseUp), p, button == 1 ? .left : button == 3 ? .right : .center) + case "key", "keyDown", "keyUp": + guard let key = action["key"] as? String else { throw DriverFailure(code: "InvalidRequest", message: "Missing key.") } + let parts = key.components(separatedBy: "+") + guard !parts.isEmpty, parts.allSatisfy({ code($0) != nil }), kind == "key" || parts.count == 1 else { + throw DriverFailure(code: "unsupported", message: "Unsupported key combination.") + } + let modifiers: [String: CGEventFlags] = ["Meta": .maskCommand, "Control": .maskControl, "Alt": .maskAlternate, "Shift": .maskShift] + if kind == "key" { + guard parts.dropLast().allSatisfy({ modifiers[$0] != nil }) else { throw DriverFailure(code: "unsupported", message: "Only modifiers may precede a key.") } + let flags = parts.dropLast().reduce(CGEventFlags()) { $0.union(modifiers[$1] ?? []) } + try keyboard(parts.last!, true, flags); try keyboard(parts.last!, false, flags) + } else { + if let modifier = modifiers[key] { + if kind == "keyDown" { desktopHeldModifiers.insert(modifier) } else { desktopHeldModifiers.remove(modifier) } + } + try keyboard(key, kind == "keyDown") + } + case "text": + guard let text = action["text"] as? String, !text.isEmpty, text.utf16.count <= 16000, !text.contains("\0") else { throw DriverFailure(code: "InvalidRequest", message: "Invalid text.") } + // Quartz limits Unicode payloads; chunk on Unicode scalar boundaries. + var chunks: [[UInt16]] = [[]] + for scalar in text.unicodeScalars { + let units = Array(String(scalar).utf16) + if chunks[chunks.count - 1].count + units.count > 20 { chunks.append([]) } + chunks[chunks.count - 1].append(contentsOf: units) + } + for units in chunks { + guard let down = CGEvent(keyboardEventSource: nil, virtualKey: 0, keyDown: true), let up = CGEvent(keyboardEventSource: nil, virtualKey: 0, keyDown: false) else { throw DriverFailure(code: "unsupported", message: "Could not allocate text event.", delivery: "unknown") } + units.withUnsafeBufferPointer { buffer in + down.keyboardSetUnicodeString(stringLength: units.count, unicodeString: buffer.baseAddress!); up.keyboardSetUnicodeString(stringLength: units.count, unicodeString: buffer.baseAddress!) + } + down.post(tap: .cghidEventTap); up.post(tap: .cghidEventTap) + } + case "scroll": + guard let amount = action["amount"] as? Int, (-30...30).contains(amount), let axis = action["axis"] as? String, ["x", "y"].contains(axis), + let event = CGEvent(scrollWheelEvent2Source: nil, units: .line, wheelCount: 2, wheel1: axis == "y" ? Int32(-amount) : 0, wheel2: axis == "x" ? Int32(-amount) : 0, wheel3: 0) else { throw DriverFailure(code: "InvalidRequest", message: "Invalid scroll.") } + event.post(tap: .cghidEventTap) + case "focus": + guard let id = action["windowId"] as? Int, let item = try desktopWindows().first(where: { $0["id"] as? Int == id }), let pid = item["pid"] as? Int else { throw DriverFailure(code: "stale_observation", message: "Window disappeared.") } + let ax = AXUIElementCreateApplication(pid_t(pid)) + guard let windows = value(ax, "AXWindows") as? [AXUIElement], let window = windows.first(where: { backgroundInput.windowID($0) == CGWindowID(id) }) else { throw DriverFailure(code: "unsupported", message: "Window is not accessible.") } + NSRunningApplication(processIdentifier: pid_t(pid))?.activate(options: []) + guard AXUIElementPerformAction(window, "AXRaise" as CFString) == .success else { throw DriverFailure(code: "ActionOutcomeUnknown", message: "Window focus unconfirmed.", delivery: "unknown") } + default: throw DriverFailure(code: "unsupported", message: "Unsupported desktop action.") + } + return ["delivery": "dispatchedUnverified"] + } + func handle(_ request: [String: Any]) async throws -> Any { switch request["method"] as? String { + case "desktopGeometry": return try desktopGeometry(request) + case "desktopWindows": return try desktopWindows() + case "desktopState": return desktopState() + case "desktopCapture": return try await desktopCapture(request) + case "desktopInput": return try desktopInput(request) case "installedApps": return installedApps().map { ["id": $0.id, "name": $0.name] } case "launchApp": guard let id = request["appId"] as? String, let target = installedApps().first(where: { $0.id == id }) else { diff --git a/src/desktop/backend.ts b/src/desktop/backend.ts index 8f902a6..25b5a03 100644 --- a/src/desktop/backend.ts +++ b/src/desktop/backend.ts @@ -1,10 +1,11 @@ +import { execFile } from "node:child_process"; import { readdir, readFile } from "node:fs/promises"; -import { join } from "node:path"; import { homedir } from "node:os"; -import { execFile } from "node:child_process"; +import { join } from "node:path"; import { promisify } from "node:util"; import { ppmToPNG } from "../linux/capture.js"; -import { failure, type Target, type Action } from "./protocol.js"; +import { type Action, failure, type Target } from "./protocol.js"; + const exec = promisify(execFile); export interface Geometry { id: number; @@ -15,7 +16,25 @@ export interface Geometry { pid?: number; title?: string; } -export class X11Backend { +export interface DesktopBackend { + readonly env: NodeJS.ProcessEnv; + readonly capabilities?: { + backend: string; + resize: string | false; + limitations: string[]; + }; + geometry(target: Target): Promise; + windows(): Promise; + apps(): Promise<{ id: string; name: string; path?: string }[]>; + state(): Promise<{ focus: number; appClass?: string; x: number; y: number }>; + capture( + target: Target, + ): Promise<{ geometry: Geometry; png: Buffer; ppm: Buffer; sample: Buffer }>; + prepareAction(action: Action): Promise; + input(target: Target, action: Action, signal: AbortSignal): Promise; + resize(width: number, height: number): Promise; +} +export class X11Backend implements DesktopBackend { constructor( readonly helper: string, readonly env: NodeJS.ProcessEnv, diff --git a/src/desktop/cli.ts b/src/desktop/cli.ts index 07b2478..3a386f3 100644 --- a/src/desktop/cli.ts +++ b/src/desktop/cli.ts @@ -1,16 +1,24 @@ -import { parseArgs } from "node:util"; -import { readFile, writeFile, mkdir, lstat, open } from "node:fs/promises"; -import { dirname, join, resolve } from "node:path"; -import { fileURLToPath } from "node:url"; import { spawn } from "node:child_process"; -import { randomUUID } from "node:crypto"; -import { Config, startDesktop, registryPath } from "./server.js"; +import { constants } from "node:fs"; +import { lstat, mkdir, open, readFile } from "node:fs/promises"; +import { dirname, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import { parseArgs } from "node:util"; +import { + desktopCall, + readCredential, + serviceURL, + writeCredential, +} from "./client.js"; import { errorJSON, failure, - methodSchema, methodArguments, + methodSchema, } from "./protocol.js"; +import { openSSHTunnel } from "./remote.js"; +import { Config, registryPath, startDesktop } from "./server.js"; + const { positionals, values } = parseArgs({ allowPositionals: true, options: { @@ -27,30 +35,25 @@ const { positionals, values } = parseArgs({ args: { type: "string" }, id: { type: "string" }, help: { type: "boolean" }, + endpoint: { type: "string" }, + ssh: { type: "string" }, + "remote-credential-file": { type: "string" }, + "local-port": { type: "string" }, + output: { type: "string" }, + subject: { type: "string" }, + role: { type: "string" }, + "ttl-ms": { type: "string" }, }, }); const command = positionals[0]; -const display = values.display ?? process.env.DISPLAY; +const display = + values.display ?? + (process.platform === "darwin" ? "macos" : process.env.DISPLAY); async function descriptor() { if (!display && !values["credential-file"]) throw failure("invalid_request", "Supply --display or --credential-file."); const path = values["credential-file"] ?? registryPath(display!); - const info = await lstat(path); - if ( - info.isSymbolicLink() || - !info.isFile() || - info.uid !== process.getuid?.() || - info.mode & 0o077 - ) - throw failure( - "permission_denied", - "Credential file must be private and owned by this user.", - ); - return JSON.parse(await readFile(path, "utf8")) as { - endpoint: string; - token: string; - generation: string; - }; + return readCredential(path); } async function call( method: string, @@ -58,47 +61,34 @@ async function call( signal?: AbortSignal, ) { const d = await descriptor(); - const endpoint = new URL(d.endpoint); - if (!["127.0.0.1", "localhost", "[::1]"].includes(endpoint.hostname)) - throw failure( - "permission_denied", - "Use a trusted local proxy for remote service access.", - ); - const request = { - id: values.id ?? randomUUID(), - generation: values.generation ?? d.generation, + return desktopCall( + { ...d, generation: values.generation ?? d.generation }, method, args, - }; - const response = await fetch(d.endpoint + "/rpc", { - method: "POST", - headers: { - Authorization: "Bearer " + d.token, - "Content-Type": "application/json", - }, - body: JSON.stringify(request), - signal: signal ?? AbortSignal.timeout(20000), - }); - const result = (await response.json()) as { - ok: boolean; - result: unknown; - error?: { code: string; message: string; delivery: string }; - }; - if (!result.ok) - throw Object.assign(new Error(result.error?.message), result.error); - return result.result; + { id: values.id, signal }, + ); } + try { if (values.help || !command) { - console.log(`Opcode display service (Linux X11, protocol 1) + console.log(`Opcode shared desktop service (macOS / Linux X11, protocol 1) cu desktop-api attach --display :N --generation HOST_GENERATION [--authority PATH] cu desktop-api serve --config PATH cu desktop-api call --display :N --method observe --args '{"target":{"kind":"display"}}' cu desktop-api mcp --display :N +cu desktop-api attach --display macos --generation HOST_GENERATION +cu desktop-api share --display macos --subject alice --role controller --output /private/alice.json [--endpoint https://mac.tailnet.ts.net] +cu desktop-api viewer --credential-file /private/alice.json +cu desktop-api tunnel --ssh user@mac --remote-credential-file /private/alice.json --output /private/local-alice.json [--local-port 4311] +Keep tunnel running; use its output credential for MCP or the viewer. +Roles: viewer (watch and cursor), controller (watch, cursor, input), agent (observe and input). +Remote credentials use HTTPS or an authenticated loopback SSH tunnel. Use --credential-file PATH for a scoped client instead of local host authority. Raw capture does not use models. attach preserves the provider display. serve owns only the broker; exit never destroys an attached display.`); } else if (command === "serve") { + if (values["credential-file"]) + throw failure("invalid_request", "serve cannot use a remote credential."); if (!values.config) throw failure("invalid_request", "Supply --config."); const config = Config.parse( JSON.parse(await readFile(values.config, "utf8")), @@ -115,6 +105,11 @@ serve owns only the broker; exit never destroys an attached display.`); for (const sig of ["SIGTERM", "SIGINT"] as const) process.once(sig, () => void service.close().then(() => process.exit(0))); } else if (command === "attach") { + if (values["credential-file"]) + throw failure( + "invalid_request", + "attach is local; use a scoped credential with call, mcp, or viewer.", + ); if (!display || !values.generation) throw failure( "invalid_request", @@ -147,9 +142,50 @@ serve owns only the broker; exit never destroys an attached display.`); const path = registryPath(display), folder = dirname(path); await mkdir(folder, { recursive: true, mode: 0o700 }); + const info = await lstat(folder); + if ( + !info.isDirectory() || + info.isSymbolicLink() || + info.uid !== process.getuid?.() || + info.mode & 0o077 + ) + throw failure("permission_denied", "Unsafe broker directory."); const configPath = path + ".config"; - await writeFile(configPath, JSON.stringify(config), { mode: 0o600 }); - const log = await open(path + ".log", "a", 0o600); + const configFile = await open( + configPath, + constants.O_WRONLY | constants.O_CREAT | constants.O_NOFOLLOW, + 0o600, + ); + try { + const metadata = await configFile.stat(); + if ( + !metadata.isFile() || + metadata.uid !== process.getuid?.() || + metadata.mode & 0o077 + ) + throw failure("permission_denied", "Unsafe broker config."); + await configFile.truncate(0); + await configFile.writeFile(JSON.stringify(config)); + } finally { + await configFile.close(); + } + const log = await open( + path + ".log", + constants.O_WRONLY | + constants.O_CREAT | + constants.O_APPEND | + constants.O_NOFOLLOW, + 0o600, + ); + const logInfo = await log.stat(); + if ( + !logInfo.isFile() || + logInfo.uid !== process.getuid?.() || + logInfo.mode & 0o077 + ) { + await log.close(); + throw failure("permission_denied", "Unsafe broker log."); + } const child = spawn( process.execPath, [fileURLToPath(import.meta.url), "serve", "--config", configPath], @@ -174,6 +210,116 @@ serve owns only the broker; exit never destroys an attached display.`); "Broker did not become ready. Check the private broker log, display, user, authority and helper path.", ); } + } else if (command === "tunnel") { + if (!values.ssh || !values["remote-credential-file"] || !values.output) + throw failure( + "invalid_request", + "Supply --ssh, --remote-credential-file and --output.", + ); + const tunnel = await openSSHTunnel({ + host: values.ssh, + remoteCredentialFile: values["remote-credential-file"], + output: values.output, + port: Number(values["local-port"] ?? 4311), + }); + console.log( + JSON.stringify({ + ready: true, + endpoint: tunnel.endpoint, + credentialFile: resolve(values.output), + }), + ); + const stop = () => void tunnel.close(); + process.once("SIGTERM", stop); + process.once("SIGINT", stop); + await tunnel.done; + await tunnel.close(); + process.off("SIGTERM", stop); + process.off("SIGINT", stop); + } else if (command === "share") { + if (!values.subject || !values.output) + throw failure( + "invalid_request", + "Supply --subject and --output for a private scoped credential.", + ); + const role = values.role ?? "viewer"; + const roles: Record = { + viewer: ["viewer-read", "presence"], + controller: ["viewer-read", "observe", "presence", "input-control"], + agent: ["observe", "viewer-read", "presence", "input-control"], + }; + if (!Object.hasOwn(roles, role)) + throw failure( + "invalid_request", + "Role must be viewer, controller, or agent.", + ); + const d = await descriptor(); + const endpoint = serviceURL(values.endpoint ?? d.endpoint).href.replace( + /\/$/, + "", + ); + const ttlMs = Number(values["ttl-ms"] ?? 3600000); + if (!Number.isInteger(ttlMs) || ttlMs < 1000 || ttlMs > 3600000) + throw failure( + "invalid_request", + "--ttl-ms must be between 1000 and 3600000.", + ); + const grant = (await call("grant", { + subject: values.subject, + scopes: roles[role], + ttlMs, + })) as { id: string; token: string; generation: string; expiresAt: number }; + try { + await writeCredential(values.output, { + endpoint, + token: grant.token, + generation: grant.generation, + }); + } catch (error) { + await call("revoke", { id: grant.id }).catch(() => {}); + throw error; + } + console.log( + JSON.stringify({ + credentialFile: resolve(values.output), + grantId: grant.id, + role, + expiresAt: grant.expiresAt, + endpoint, + }), + ); + } else if (command === "viewer") { + if (!values["credential-file"]) + throw failure( + "permission_denied", + "Use a scoped --credential-file created by share, never the host descriptor.", + ); + const d = await descriptor(); + const sessionResponse = await fetch( + d.endpoint.replace(/\/$/, "") + "/session", + { + headers: { Authorization: "Bearer " + d.token }, + redirect: "error", + signal: AbortSignal.timeout(10000), + }, + ); + const session = (await sessionResponse.json()) as { + admin?: boolean; + scopes?: string[]; + }; + if ( + !sessionResponse.ok || + session.admin || + !session.scopes?.includes("viewer-read") + ) + throw failure( + "permission_denied", + "A scoped viewer credential is required.", + ); + await desktopCall(d, "presence.list", {}); + console.log( + d.endpoint.replace(/\/$/, "") + "/view#" + encodeURIComponent(d.token), + ); } else if (command === "call") { if (!values.method) throw failure("invalid_request", "Supply --method."); console.log( @@ -192,6 +338,7 @@ serve owns only the broker; exit never destroys an attached display.`); import("@modelcontextprotocol/sdk/types.js"), ]); const methods = [ + "presence.list", "health", "apps", "windows", @@ -212,6 +359,7 @@ serve owns only the broker; exit never destroys an attached display.`); "recording.delete", ]; const reads = new Set([ + "presence.list", "health", "apps", "windows", @@ -241,21 +389,85 @@ serve owns only the broker; exit never destroys an attached display.`); })), })); let ownedLease: string | undefined; + let participant: { id: string } | undefined; + let presenceEnabled = true; + let presenceBusy = false; + let closing = false; + const observations = new Map< + string, + { target: unknown; image: { width: number; height: number } } + >(); + async function heartbeat() { + if (closing || !presenceEnabled || presenceBusy) return; + presenceBusy = true; + try { + if (!participant) + participant = (await call("presence.join", { role: "agent" })) as { + id: string; + }; + else await call("presence.update", { participantId: participant.id }); + } catch (error) { + if ((error as { code?: string }).code === "permission_denied") + presenceEnabled = false; + participant = undefined; + } finally { + presenceBusy = false; + } + } + await heartbeat(); + const presenceTimer = setInterval(() => void heartbeat(), 4000); + presenceTimer.unref(); server.setRequestHandler(CallToolRequestSchema, async (request, extra) => { const method = methods.find( (m) => "computer_" + m.replaceAll(".", "_") === request.params.name, ); try { if (!method) throw failure("unsupported", "Unknown tool."); - const result = await call( - method, - request.params.arguments ?? {}, - extra.signal, - ); + const args = { ...request.params.arguments }; + if ( + (method === "acquire" || method === "takeover") && + participant && + !args.participantId + ) + args.participantId = participant.id; + if ( + method === "input" && + participant && + typeof args.observationId === "string" + ) { + const observation = observations.get(args.observationId); + const action = args.action as { x?: number; y?: number } | undefined; + if ( + observation && + typeof action?.x === "number" && + typeof action.y === "number" + ) { + await call("presence.update", { + participantId: participant.id, + cursor: { + x: action.x / observation.image.width, + y: action.y / observation.image.height, + target: observation.target, + }, + }).catch(() => {}); + } + } + const result = await call(method, args, extra.signal); if (method === "acquire" || method === "takeover") ownedLease = (result as { id: string }).id; if (method === "release" || method === "stop") ownedLease = undefined; if (method === "observe") { + const observed = result as { + observationId: string; + target: unknown; + image: { width: number; height: number }; + }; + observations.set(observed.observationId, { + target: observed.target, + image: observed.image, + }); + if (observations.size > 32) + observations.delete(observations.keys().next().value!); const r = result as { image: { base64: string; mimeType: string }; [key: string]: unknown; @@ -290,6 +502,12 @@ serve owns only the broker; exit never destroys an attached display.`); } }); server.onclose = () => { + closing = true; + clearInterval(presenceTimer); + if (participant) + void call("presence.leave", { participantId: participant.id }).catch( + () => {}, + ); if (ownedLease) void call("release", { leaseId: ownedLease }).catch(() => {}); }; diff --git a/src/desktop/client.ts b/src/desktop/client.ts new file mode 100644 index 0000000..65208d2 --- /dev/null +++ b/src/desktop/client.ts @@ -0,0 +1,161 @@ +import { randomUUID } from "node:crypto"; +import { constants } from "node:fs"; +import { mkdir, open, unlink } from "node:fs/promises"; +import { dirname } from "node:path"; +import { z } from "zod"; +import { failure } from "./protocol.js"; + +export const Credential = z.object({ + endpoint: z.string().url(), + token: z.string().min(16).max(4096), + generation: z.string().min(1).max(128), +}); +export type Credential = z.infer; + +export function serviceURL(endpoint: string) { + const url = new URL(endpoint); + const local = ["127.0.0.1", "localhost", "[::1]"].includes(url.hostname); + if ( + (url.protocol !== "https:" && !(local && url.protocol === "http:")) || + url.username || + url.password || + url.search || + url.hash || + !/^(?:\/[a-zA-Z0-9_-]+)*\/?$/.test(url.pathname) + ) + throw failure( + "permission_denied", + "Use HTTPS for remote desktops or HTTP through a loopback SSH tunnel; credentials cannot be embedded in the endpoint.", + ); + return url; +} + +export async function readCredential(path: string): Promise { + const file = await open(path, constants.O_RDONLY | constants.O_NOFOLLOW); + try { + const info = await file.stat(); + if ( + !info.isFile() || + info.uid !== process.getuid?.() || + info.mode & 0o077 || + info.size > 16384 + ) + throw failure( + "permission_denied", + "Credential file must be private, bounded, and owned by this user.", + ); + const value = Credential.parse(JSON.parse(await file.readFile("utf8"))); + serviceURL(value.endpoint); + return value; + } finally { + await file.close(); + } +} + +export async function writeCredential(path: string, value: Credential) { + Credential.parse(value); + serviceURL(value.endpoint); + await mkdir(dirname(path), { recursive: true, mode: 0o700 }); + const file = await open( + path, + constants.O_WRONLY | + constants.O_CREAT | + constants.O_EXCL | + constants.O_NOFOLLOW, + 0o600, + ); + try { + await file.writeFile(JSON.stringify(value) + "\n"); + } catch (error) { + await unlink(path).catch(() => {}); + throw error; + } finally { + await file.close(); + } +} + +const reads = new Set([ + "health", + "apps", + "windows", + "state", + "observe", + "recording.list", + "presence.list", +]); +export async function desktopCall( + credential: Credential, + method: string, + args: Record, + options: { id?: string; signal?: AbortSignal; fetch?: typeof fetch } = {}, +): Promise { + const endpoint = serviceURL(credential.endpoint).href.replace(/\/$/, ""); + options.signal?.throwIfAborted(); + let response: Response; + try { + response = await (options.fetch ?? fetch)(endpoint + "/rpc", { + method: "POST", + redirect: "error", + headers: { + Authorization: "Bearer " + credential.token, + "Content-Type": "application/json", + }, + body: JSON.stringify({ + id: options.id ?? randomUUID(), + generation: credential.generation, + method, + args, + }), + signal: options.signal + ? AbortSignal.any([options.signal, AbortSignal.timeout(20000)]) + : AbortSignal.timeout(20000), + }); + } catch { + throw failure( + "connection_lost", + reads.has(method) + ? "Desktop connection failed." + : "Desktop connection failed; the action may have been delivered. Observe before deciding whether to retry.", + reads.has(method) ? "notDispatched" : "unknown", + ); + } + let result: { + ok: boolean; + protocol?: number; + generation?: string; + result?: unknown; + error?: { code?: string; message?: string; delivery?: string }; + }; + try { + result = (await response.json()) as typeof result; + } catch { + throw failure( + "connection_lost", + "Desktop returned an incomplete response. Do not automatically repeat input.", + reads.has(method) ? "notDispatched" : "unknown", + ); + } + if (!result || typeof result !== "object") + throw failure( + "invalid_response", + "Invalid desktop response.", + reads.has(method) ? "notDispatched" : "unknown", + ); + if (!result.ok || !response.ok) + throw failure( + result.error?.code ?? "connection_failed", + result.error?.message ?? "Desktop request failed.", + result.error?.delivery ?? + (reads.has(method) ? "notDispatched" : "unknown"), + ); + if ( + result.protocol !== 1 || + (result.generation !== credential.generation && method !== "rebind") + ) + throw failure( + "generation_mismatch", + "Desktop protocol or lifecycle changed. Reconnect with a fresh credential.", + reads.has(method) ? "notDispatched" : "unknown", + ); + return result.result; +} diff --git a/src/desktop/controller.ts b/src/desktop/controller.ts index 6afe8e4..0f6efb4 100644 --- a/src/desktop/controller.ts +++ b/src/desktop/controller.ts @@ -1,35 +1,41 @@ -import { readFile, writeFile, rename } from "node:fs/promises"; +import { createHash, randomUUID } from "node:crypto"; +import { readFile, rename, writeFile } from "node:fs/promises"; import { join } from "node:path"; -import { randomUUID, createHash } from "node:crypto"; import { z } from "zod"; -import { X11Backend, type Geometry } from "./backend.js"; import { Authority, type Grant } from "./authority.js"; +import type { DesktopBackend, Geometry } from "./backend.js"; +import { Presence } from "./presence.js"; import { Action, - methodArguments, - Target, failure, + methodArguments, type Observation, type Request, scopes, + Target, } from "./protocol.js"; import { Recorder } from "./recording.js"; + type Lease = { id: string; grantId: string; + participantId?: string; subject: string; owner: "human" | "agent"; epoch: number; expiresAt: number; }; -type Frame = Awaited>; +type Frame = Awaited>; export class DesktopController { readonly authority: Authority; readonly recorder: Recorder; + readonly presence: Presence; private epoch = 1; private geometryKey = ""; private lease?: Lease; private fenced = false; + private closed = false; + private retryFenceAt = 0; private observations = new Map< string, { @@ -54,17 +60,23 @@ export class DesktopController { private timer: ReturnType; private recorderGrant?: string; constructor( - readonly backend: X11Backend, + readonly backend: DesktopBackend, generation: string, private directory: string, ) { this.authority = new Authority(generation); + this.presence = new Presence(this.authority); this.recorder = new Recorder(directory, (t) => this.capture(t)); this.timer = setInterval(() => { + this.presence.sweep(); + if (this.fenced && !this.fenceTask && Date.now() >= this.retryFenceAt) + void this.fence().catch(() => {}); if ( this.lease && (this.lease.expiresAt <= Date.now() || - !this.authority.valid(this.lease.grantId)) + !this.authority.valid(this.lease.grantId) || + (this.lease.participantId !== undefined && + !this.presence.has(this.lease.participantId))) ) void this.fence().catch(() => {}); if (this.recorderGrant && !this.authority.valid(this.recorderGrant)) { @@ -185,7 +197,21 @@ export class DesktopController { epoch: this.epoch, sample, }); - while (this.observations.size > 32) + // Bound each credential independently so one fast viewer cannot evict every + // other participant's observation before their input reaches the broker. + let grantCount = 0; + for (const record of this.observations.values()) + if (record.grantId === grant.id) grantCount++; + for (const [id, record] of this.observations) { + if ( + performance.now() - record.at > 30000 || + (record.grantId === grant.id && grantCount > 8) + ) { + this.observations.delete(id); + if (record.grantId === grant.id) grantCount--; + } + } + while (this.observations.size > 512) this.observations.delete(this.observations.keys().next().value!); return value; } @@ -193,47 +219,72 @@ export class DesktopController { async fence() { if (this.fenceTask) return this.fenceTask; this.fenced = true; + this.retryFenceAt = Date.now() + 1000; this.epoch++; this.lease = undefined; this.observations.clear(); this.active?.abort.abort(); const task = (async () => { await this.active?.done.catch(() => {}); - for (const key of this.heldKeys) - await this.backend - .input( - { kind: "display" }, - { kind: "keyUp", key }, - new AbortController().signal, - ) - .catch(() => {}); - for (const button of this.heldButtons) - await this.backend - .input( - { kind: "display" }, - { kind: "buttonUp", button }, - new AbortController().signal, - ) - .catch(() => {}); - this.heldKeys.clear(); - this.heldButtons.clear(); - await this.persistHeld(); + const abort = new AbortController(); + const deadline = setTimeout(() => abort.abort(), 10000); + try { + for (const key of this.heldKeys) { + if (abort.signal.aborted) break; + try { + await this.backend.input( + { kind: "display" }, + { kind: "keyUp", key }, + abort.signal, + ); + this.heldKeys.delete(key); + } catch { + /* Preserve failed releases in the recovery journal. */ + } + } + for (const button of this.heldButtons) { + if (abort.signal.aborted) break; + try { + await this.backend.input( + { kind: "display" }, + { kind: "buttonUp", button }, + abort.signal, + ); + this.heldButtons.delete(button); + } catch { + /* Preserve failed releases in the recovery journal. */ + } + } + await this.persistHeld(); + if (this.heldKeys.size || this.heldButtons.size) + throw failure( + "input_cleanup_failed", + "Held input could not be released. Control remains blocked until cleanup or service recovery succeeds.", + "unknown", + ); + this.fenced = false; + } finally { + clearTimeout(deadline); + } })(); this.fenceTask = task; try { await task; } finally { this.fenceTask = undefined; - this.fenced = false; } } private requireLease(grant: Grant, id: unknown) { if ( + this.closed || this.fenced || !this.lease || this.lease.id !== id || this.lease.grantId !== grant.id || - this.lease.expiresAt <= Date.now() + !this.authority.valid(grant.id) || + this.lease.expiresAt <= Date.now() || + (this.lease.participantId !== undefined && + !this.presence.has(this.lease.participantId)) ) throw failure( "lease_revoked", @@ -367,7 +418,7 @@ export class DesktopController { finish(); this.active = undefined; this.frame = undefined; - if (failed) await this.fence(); + if (failed) await this.fence().catch(() => {}); } }) .finally(() => this.queued--); @@ -375,14 +426,22 @@ export class DesktopController { return job; } async call(request: Request, grant: Grant, admin = false): Promise { + if (this.closed) + throw failure("service_closed", "Desktop service is closing."); if (request.generation !== this.generation) throw failure("generation_mismatch", "Host rebind required."); + if (!admin && !this.authority.valid(grant.id)) + throw failure("permission_denied", "Credential expired or revoked."); const methods: Record = { health: "observe", apps: "observe", windows: "observe", observe: "observe", state: "viewer-read", + "presence.list": "viewer-read", + "presence.join": "presence", + "presence.update": "presence", + "presence.leave": "presence", acquire: "input-control", renew: "input-control", release: "input-control", @@ -407,6 +466,8 @@ export class DesktopController { "observe", "health", "state", + "presence.list", + "presence.update", "windows", "apps", "recording.list", @@ -441,16 +502,24 @@ export class DesktopController { displayId: this.backend.env.DISPLAY ?? "unknown", generation: this.generation, displayEpoch: this.epoch, - backend: "x11", + backend: this.backend.capabilities?.backend ?? "x11", mode: "attach", geometry: frame.geometry, rawCapture: true, perception: false, recording: await this.recorder.available(), input: true, - resize: "randr-dependent", + multiplayer: { + presence: true, + cursorSpace: "normalized-target", + maxParticipants: 64, + heartbeatMs: 5000, + participantTtlMs: 15000, + inputOwners: 1, + }, + resize: this.backend.capabilities?.resize ?? "randr-dependent", coordinateSpace: "image-pixels", - limitations: [ + limitations: this.backend.capabilities?.limitations ?? [ "No Wayland, audio or synchronized multi-monitor.", "External X clients bypass application arbitration.", ], @@ -465,15 +534,30 @@ export class DesktopController { return this.backend.windows(); case "observe": return this.observe(g, Target.parse(a.target ?? { kind: "display" })); + case "presence.join": + return this.presence.join( + g, + a.name as string | undefined, + a.role as "human" | "agent", + ); + case "presence.update": + return this.presence.update( + g, + a.participantId as string, + a.cursor as { x: number; y: number } | null | undefined, + ); + case "presence.leave": { + const id = a.participantId as string; + this.presence.leave(g, id); + if (this.lease?.participantId === id) await this.fence(); + return { left: true }; + } + case "presence.list": + return this.multiplayerState(); case "state": return { - lease: this.lease - ? { - owner: this.lease.owner, - subject: this.lease.subject, - expiresAt: this.lease.expiresAt, - } - : null, + ...this.multiplayerState(), + lease: this.controllerState(), displayEpoch: this.epoch, recordings: this.recorder .list() @@ -481,6 +565,8 @@ export class DesktopController { }; case "acquire": case "takeover": { + const participantId = a.participantId as string | undefined; + if (participantId) this.presence.require(g, participantId); if (this.fenced) throw failure("lease_conflict", "Control transition is in progress."); if ( @@ -490,12 +576,14 @@ export class DesktopController { ) throw failure("lease_conflict", "Display already has a controller."); await this.fence(); - if (!this.authority.valid(g.id) && !admin) + if (this.closed || (!this.authority.valid(g.id) && !admin)) throw failure( "permission_denied", "Credential expired during takeover.", ); + if (participantId) this.presence.require(g, participantId); this.lease = { + participantId, id: randomUUID(), grantId: g.id, subject: g.subject, @@ -571,6 +659,7 @@ export class DesktopController { await this.fence(); await this.recorder.fence(); this.authority.rebind(next); + this.presence.clear(); this.frame = undefined; this.cache.clear(); return { generation: next }; @@ -623,12 +712,32 @@ export class DesktopController { } throw failure("unsupported", "Unsupported desktop operation."); } + private controllerState() { + if ( + !this.lease || + this.lease.expiresAt <= Date.now() || + !this.authority.valid(this.lease.grantId) || + (this.lease.participantId !== undefined && + !this.presence.has(this.lease.participantId)) + ) + return null; + const { owner, subject, participantId, expiresAt } = this.lease; + return { owner, subject, participantId, expiresAt }; + } + multiplayerState() { + return { + participants: this.presence.list(), + controller: this.controllerState(), + }; + } async cancelGrantInput(grantId: string) { if (this.lease?.grantId === grantId) await this.fence(); } async close() { + if (this.closed) return; + this.closed = true; clearInterval(this.timer); - await this.fence(); - await this.recorder.fence(); + this.presence.clear(); + await Promise.allSettled([this.fence(), this.recorder.fence()]); } } diff --git a/src/desktop/index.ts b/src/desktop/index.ts index 8bac0b6..f92496f 100644 --- a/src/desktop/index.ts +++ b/src/desktop/index.ts @@ -1,11 +1,19 @@ -export { Config, startDesktop, registryPath } from "./server.js"; +export type { Credential } from "./client.js"; +export { + desktopCall, + readCredential, + serviceURL, + writeCredential, +} from "./client.js"; +export type { Observation, Participant, Scope } from "./protocol.js"; export { - Request, Action, - Target, - scopes, - VERSION, methodArguments, methodSchema, + Request, + scopes, + Target, + VERSION, } from "./protocol.js"; -export type { Observation, Scope } from "./protocol.js"; +export { openSSHTunnel } from "./remote.js"; +export { Config, registryPath, startDesktop } from "./server.js"; diff --git a/src/desktop/mac-backend.ts b/src/desktop/mac-backend.ts new file mode 100644 index 0000000..7bedc58 --- /dev/null +++ b/src/desktop/mac-backend.ts @@ -0,0 +1,112 @@ +import { z } from "zod"; +import type { DesktopBackend, Geometry } from "./backend.js"; +import { type Action, failure, type Target } from "./protocol.js"; + +export interface MacDriver { + request( + method: string, + args?: Record, + signal?: AbortSignal, + ): Promise; + close?(): void; +} +const GeometrySchema = z.object({ + id: z.number().int().positive(), + x: z.number().finite(), + y: z.number().finite(), + width: z.number().int().positive().max(8192), + height: z.number().int().positive().max(8192), + pid: z.number().int().positive().optional(), + title: z.string().optional(), +}); +export class MacBackend implements DesktopBackend { + readonly env = { DISPLAY: "macos" }; + readonly capabilities = { + backend: "macos", + resize: false as const, + limitations: [ + "Requires macOS 14+, Screen Recording and Accessibility permissions for Opcode.", + "Primary display only; physical input is serialized, participant cursors are overlays.", + "Local applications and physical input bypass broker arbitration.", + ], + }; + constructor(readonly driver: MacDriver) {} + async geometry(target: Target): Promise { + return GeometrySchema.parse( + await this.driver.request("desktopGeometry", { target }), + ); + } + async windows(): Promise { + return z + .array(GeometrySchema) + .parse(await this.driver.request("desktopWindows")); + } + async apps() { + return z + .array(z.object({ id: z.string(), name: z.string() })) + .parse(await this.driver.request("installedApps")); + } + async state() { + return z + .object({ + focus: z.number().int(), + x: z.number().finite(), + y: z.number().finite(), + }) + .parse(await this.driver.request("desktopState")); + } + async capture(target: Target) { + const result = z + .object({ + geometry: GeometrySchema, + base64: z.string().max(16_000_000), + sample: z.string().max(16_384), + }) + .parse(await this.driver.request("desktopCapture", { target })); + const png = Buffer.from(result.base64, "base64"); + const sample = Buffer.from(result.sample, "base64"); + if ( + png.length < 24 || + !png + .subarray(0, 8) + .equals(Buffer.from([137, 80, 78, 71, 13, 10, 26, 10])) || + png.readUInt32BE(16) !== result.geometry.width || + png.readUInt32BE(20) !== result.geometry.height || + sample.length !== 64 * 48 * 4 + ) + throw failure( + "display_unavailable", + "Mac helper returned an invalid desktop frame.", + ); + return { geometry: result.geometry, png, ppm: Buffer.alloc(0), sample }; + } + async prepareAction(action: Action) { + return action; + } + async input(target: Target, action: Action, signal: AbortSignal) { + signal.throwIfAborted(); + if (action.kind === "launch") { + if (!(await this.apps()).some((app) => app.id === action.appId)) + throw failure("not_found", "Unknown installed app ID."); + signal.throwIfAborted(); + await this.driver.request("launchApp", { appId: action.appId }, signal); + return; + } + // Releases must remain possible after the originally observed window disappears. + if (action.kind === "keyUp" || action.kind === "buttonUp") + target = { kind: "display" }; + const geometry = await this.geometry(target); + signal.throwIfAborted(); + await this.driver.request( + "desktopInput", + { target, action, expectedGeometry: geometry, foregroundApproved: true }, + signal, + ); + } + async resize(_width: number, _height: number): Promise { + throw failure( + "unsupported", + "Resizing a physical Mac display is not supported.", + ); + } +} diff --git a/src/desktop/mac-driver.ts b/src/desktop/mac-driver.ts new file mode 100644 index 0000000..ee00106 --- /dev/null +++ b/src/desktop/mac-driver.ts @@ -0,0 +1,164 @@ +import { randomUUID } from "node:crypto"; +import { lstat } from "node:fs/promises"; +import { connect, type Socket } from "node:net"; +import type { MacDriver } from "./mac-backend.js"; +import { failure } from "./protocol.js"; + +/** Connect only to the installed LaunchServices helper; never spawn an untrusted binary. */ +export async function connectMacDriver(): Promise { + const uid = process.getuid?.(); + if (uid === undefined) + throw failure("unsupported", "Mac helper requires a Unix user."); + const directory = `/tmp/opcode-cu-${uid}`; + for (const path of [directory, `${directory}/driver.sock`]) { + const stat = await lstat(path).catch(() => { + throw failure( + "display_unavailable", + "Start the installed Opcode helper with cu install, then grant its permissions.", + ); + }); + if ( + stat.isSymbolicLink() || + stat.uid !== uid || + stat.mode & 0o077 || + (path === directory ? !stat.isDirectory() : !stat.isSocket()) + ) + throw failure( + "permission_denied", + "Unsafe Mac helper socket permissions.", + ); + } + const socket = await new Promise((resolve, reject) => { + const socket = connect(`${directory}/driver.sock`); + const timeout = setTimeout( + () => socket.destroy(new Error("Mac helper connection timed out.")), + 3000, + ); + socket.once("error", reject); + socket.once("connect", () => { + clearTimeout(timeout); + socket.off("error", reject); + resolve(socket); + }); + socket.once("close", () => clearTimeout(timeout)); + }); + return socketMacDriver(socket); +} + +/** Exported for transport tests. A disconnected session never reconnects or replays input. */ +export function socketMacDriver(socket: Socket): MacDriver { + let chunks: Buffer[] = []; + let bufferedBytes = 0; + let closed = false; + let tail: Promise = Promise.resolve(); + let pending: + | { + id: string; + resolve(value: unknown): void; + reject(error: Error): void; + timer: ReturnType; + } + | undefined; + const disconnect = () => { + closed = true; + if (pending) { + clearTimeout(pending.timer); + pending.reject( + failure( + "driver_disconnected", + "Mac helper disconnected. Observe before retrying; input delivery is unknown.", + "unknown", + ), + ); + pending = undefined; + } + socket.destroy(); + }; + socket.on("error", disconnect); + socket.on("close", disconnect); + socket.on("data", (chunk: Buffer) => { + let offset = 0; + while (offset < chunk.length) { + const index = chunk.indexOf(10, offset); + const part = chunk.subarray(offset, index < 0 ? chunk.length : index); + chunks.push(part); + bufferedBytes += part.length; + if (bufferedBytes > 17_000_000) { + disconnect(); + return; + } + if (index < 0) return; + const line = Buffer.concat(chunks, bufferedBytes); + chunks = []; + bufferedBytes = 0; + offset = index + 1; + try { + const reply = JSON.parse(line.toString()); + if ( + !pending || + reply.id !== pending.id || + typeof reply.ok !== "boolean" + ) { + disconnect(); + return; + } + const task = pending; + pending = undefined; + clearTimeout(task.timer); + if (reply.ok) task.resolve(reply.data); + else + task.reject( + failure( + reply.error?.code ?? "driver_error", + reply.error?.message ?? "Mac helper request failed.", + reply.error?.delivery ?? "unknown", + ), + ); + } catch { + disconnect(); + return; + } + } + }); + return { + close: disconnect, + request(method, args = {}, signal) { + const task = tail + .catch(() => {}) + .then(async () => { + signal?.throwIfAborted(); + if (closed) + throw failure( + "driver_disconnected", + "Mac helper is disconnected. Restart the desktop service.", + ); + const id = randomUUID(); + const wire = JSON.stringify({ ...args, id, method }) + "\n"; + if (Buffer.byteLength(wire) > 128_000) + throw failure( + "invalid_request", + "Mac helper request exceeds limit.", + ); + const result = await new Promise((resolve, reject) => { + pending = { + id, + resolve, + reject, + timer: setTimeout(disconnect, 12_000), + }; + socket.write(wire); + }); + // Wait for completion before fencing a cancelled in-flight action. Never retry it. + if (signal?.aborted) + throw failure( + "cancelled", + "Mac operation finished after cancellation; verify fresh state.", + "unknown", + ); + return result; + }); + tail = task; + return task; + }, + }; +} diff --git a/src/desktop/presence.ts b/src/desktop/presence.ts new file mode 100644 index 0000000..3cbd0c4 --- /dev/null +++ b/src/desktop/presence.ts @@ -0,0 +1,104 @@ +import { randomUUID } from "node:crypto"; +import type { Authority, Grant } from "./authority.js"; +import { failure, type Participant } from "./protocol.js"; + +/** Visual cursors are presence only. Actual OS input remains lease-controlled. */ +export class Presence { + private entries = new Map< + string, + { grantId: string; value: Participant; updatedAt: number } + >(); + constructor( + private authority: Authority, + private now = () => Date.now(), + ) {} + sweep() { + const removed: string[] = []; + for (const [id, entry] of this.entries) { + if ( + entry.value.expiresAt <= this.now() || + !this.authority.valid(entry.grantId) + ) { + this.entries.delete(id); + removed.push(id); + } + } + return removed; + } + list(): Participant[] { + this.sweep(); + return [...this.entries.values()].map(({ value }) => + structuredClone(value), + ); + } + has(id: string) { + this.sweep(); + return this.entries.has(id); + } + require(grant: Grant, id: string) { + this.sweep(); + const entry = this.entries.get(id); + if (!entry || entry.grantId !== grant.id) + throw failure( + "participant_expired", + "Participant expired or belongs to another credential; join again.", + ); + return entry; + } + join( + grant: Grant, + name: string | undefined, + role: Participant["role"], + ): Participant { + this.sweep(); + if ( + this.entries.size >= 64 || + [...this.entries.values()].filter((e) => e.grantId === grant.id).length >= + 8 + ) + throw failure("overloaded", "Participant limit reached."); + const id = randomUUID(); + const colors = [ + "#60a5fa", + "#f472b6", + "#34d399", + "#fbbf24", + "#a78bfa", + "#fb923c", + ]; + const value: Participant = { + id, + subject: grant.subject, + name: name ?? grant.subject.slice(0, 64), + role, + color: colors[parseInt(id.slice(0, 8), 16) % colors.length] ?? "#60a5fa", + cursor: null, + expiresAt: Math.min(grant.expiresAt, this.now() + 15000), + }; + this.entries.set(id, { grantId: grant.id, value, updatedAt: -Infinity }); + return structuredClone(value); + } + update( + grant: Grant, + id: string, + cursor: Participant["cursor"] | undefined, + ): Participant { + const entry = this.require(grant, id); + if (this.now() - entry.updatedAt < 40) + throw failure( + "overloaded", + "Presence updates are limited to 25 per second.", + ); + entry.updatedAt = this.now(); + if (cursor !== undefined) entry.value.cursor = cursor; + entry.value.expiresAt = Math.min(grant.expiresAt, this.now() + 15000); + return structuredClone(entry.value); + } + leave(grant: Grant, id: string) { + this.require(grant, id); + this.entries.delete(id); + } + clear() { + this.entries.clear(); + } +} diff --git a/src/desktop/protocol.ts b/src/desktop/protocol.ts index ab47404..d2d9591 100644 --- a/src/desktop/protocol.ts +++ b/src/desktop/protocol.ts @@ -3,6 +3,7 @@ export const VERSION = 1; export const scopes = [ "observe", "viewer-read", + "presence", "input-control", "recording-start", "recording-read", @@ -61,7 +62,11 @@ export const Action = z.discriminatedUnion("kind", [ }), z.object({ kind: z.literal("launch"), - appId: z.string().regex(/^[a-zA-Z0-9_.-]+\.desktop$/), + appId: z + .string() + .min(1) + .max(256) + .regex(/^[a-zA-Z0-9_.-]+$/), }), z.object({ kind: z.literal("focus"), windowId: z.number().int().positive() }), ]); @@ -108,6 +113,21 @@ export interface Observation { } const leaseId = z.string().min(1); +export const Cursor = z.object({ + target: Target.optional(), + x: z.number().finite().min(0).max(1), + y: z.number().finite().min(0).max(1), +}); +export interface Participant { + id: string; + subject: string; + name: string; + role: "human" | "agent"; + color: string; + cursor: z.infer | null; + expiresAt: number; +} +const participantId = z.string().uuid(); const recordingId = z.object({ id: z.string().uuid() }); /** Tool metadata and runtime argument validation share these definitions. */ export const methodArguments = { @@ -116,10 +136,22 @@ export const methodArguments = { windows: z.object({}), state: z.object({}), observe: z.object({ target: Target.default({ kind: "display" }) }), + "presence.join": z.object({ + name: z.string().trim().min(1).max(64).optional(), + role: z.enum(["human", "agent"]).default("human"), + }), + "presence.update": z.object({ + participantId, + cursor: Cursor.nullable().optional(), + }), + "presence.leave": z.object({ participantId }), + "presence.list": z.object({}), acquire: z.object({ + participantId: participantId.optional(), ttlMs: z.number().int().min(1000).max(60000).default(30000), }), takeover: z.object({ + participantId: participantId.optional(), ttlMs: z.number().int().min(1000).max(60000).default(30000), }), renew: z.object({ leaseId }), diff --git a/src/desktop/registry.ts b/src/desktop/registry.ts new file mode 100644 index 0000000..5e153ca --- /dev/null +++ b/src/desktop/registry.ts @@ -0,0 +1,25 @@ +import { createHash } from "node:crypto"; +import { homedir } from "node:os"; +import { join } from "node:path"; +import { + canonical as canonicalX11, + displayKey as x11Key, + registryPath as x11Path, +} from "../linux/display-registry.js"; +export const canonical = (display: string) => + display === "macos" ? display : canonicalX11(display); +export const displayKey = (display: string) => + display === "macos" + ? createHash("sha256") + .update(`${process.getuid?.()}:macos`) + .digest("hex") + .slice(0, 24) + : x11Key(display); +export const registryPath = (display: string) => + display === "macos" + ? join( + homedir(), + ".local/state/opcode/displays", + displayKey(display) + ".json", + ) + : x11Path(display); diff --git a/src/desktop/remote.ts b/src/desktop/remote.ts new file mode 100644 index 0000000..b5ccce8 --- /dev/null +++ b/src/desktop/remote.ts @@ -0,0 +1,207 @@ +import { execFile, spawn } from "node:child_process"; +import { unlink } from "node:fs/promises"; +import { promisify } from "node:util"; +import { + Credential, + desktopCall, + serviceURL, + writeCredential, +} from "./client.js"; +import { failure } from "./protocol.js"; + +const exec = promisify(execFile); + +export function sshTarget(value: string) { + if (!/^(?:[a-zA-Z0-9_.-]+@)?[a-zA-Z0-9][a-zA-Z0-9_.-]*$/.test(value)) + throw failure( + "invalid_request", + "Use an SSH config alias or user@hostname.", + ); + return value; +} +export function remoteCredentialCommand(path: string) { + if (!path.startsWith("/") || path.length > 4096 || /[\0\r\n]/.test(path)) + throw failure( + "invalid_request", + "Remote credential path must be absolute.", + ); + return "cat -- '" + path.replaceAll("'", "'\\''") + "'"; +} + +/** OpenSSH retains ownership of keys, host verification, proxies and Tailnet routing. */ +export async function openSSHTunnel(options: { + host: string; + remoteCredentialFile: string; + output: string; + port: number; +}) { + const host = sshTarget(options.host); + if ( + !Number.isInteger(options.port) || + options.port < 1024 || + options.port > 65535 + ) + throw failure( + "invalid_request", + "Local port must be between 1024 and 65535.", + ); + const common = [ + "-T", + "-o", + "BatchMode=yes", + "-o", + "ConnectTimeout=10", + "-o", + "StrictHostKeyChecking=yes", + ]; + let remote: Credential; + try { + const { stdout } = await exec( + "ssh", + [ + ...common, + "--", + host, + remoteCredentialCommand(options.remoteCredentialFile), + ], + { timeout: 15000, maxBuffer: 16384 }, + ); + remote = Credential.parse(JSON.parse(stdout)); + } catch { + throw failure( + "connection_failed", + "Could not read the scoped remote credential. Check the SSH alias, accepted host key, non-interactive authentication and absolute credential path.", + ); + } + const endpoint = serviceURL(remote.endpoint); + if ( + endpoint.protocol !== "http:" || + !["localhost", "127.0.0.1"].includes(endpoint.hostname) + ) + throw failure( + "invalid_request", + "SSH sharing requires a remote loopback HTTP credential. For HTTPS, use the credential directly.", + ); + const port = endpoint.port || "80"; + const local: Credential = { + ...remote, + endpoint: `http://127.0.0.1:${options.port}${endpoint.pathname.replace(/\/$/, "")}`, + }; + const child = spawn( + "ssh", + [ + ...common, + "-N", + "-o", + "PermitLocalCommand=yes", + "-o", + "LocalCommand=printf OPCODE_TUNNEL_READY", + "-o", + "ExitOnForwardFailure=yes", + "-o", + "ServerAliveInterval=10", + "-o", + "ServerAliveCountMax=3", + "-L", + `127.0.0.1:${options.port}:127.0.0.1:${port}`, + "--", + host, + ], + { stdio: ["ignore", "pipe", "ignore"] }, + ); + let exited = false; + const readySignal = new Promise((resolve, reject) => { + let output = ""; + const timer = setTimeout( + () => + reject( + failure( + "connection_failed", + "SSH did not confirm forwarding readiness.", + ), + ), + 15000, + ); + const finish = () => { + clearTimeout(timer); + }; + child.stdout?.on("data", (chunk) => { + output += chunk.toString(); + if (output.includes("OPCODE_TUNNEL_READY")) { + finish(); + resolve(); + } else if (output.length > 4096) { + finish(); + reject(failure("connection_failed", "Unexpected SSH startup output.")); + } + }); + child.once("error", () => { + finish(); + reject(failure("connection_failed", "SSH could not start.")); + }); + child.once("exit", () => { + finish(); + reject( + failure( + "connection_failed", + "SSH closed before forwarding became ready.", + ), + ); + }); + }); + let written = false; + let cleaned = false; + const done = new Promise((resolve) => { + child.once("error", () => { + exited = true; + resolve(); + }); + child.once("exit", () => { + exited = true; + resolve(); + }); + }); + const close = async () => { + if (cleaned) return; + cleaned = true; + child.kill("SIGTERM"); + const timer = setTimeout(() => child.kill("SIGKILL"), 1000); + timer.unref(); + await done; + clearTimeout(timer); + if (written) await unlink(options.output).catch(() => {}); + }; + try { + await readySignal; + let ready = false; + for (let i = 0; i < 50 && !exited; i++) { + try { + await desktopCall( + local, + "presence.list", + {}, + { signal: AbortSignal.timeout(1000) }, + ); + ready = true; + break; + } catch (error) { + if ((error as { code?: string }).code !== "connection_lost") + throw error; + } + await new Promise((resolve) => setTimeout(resolve, 100)); + } + if (!ready || exited) + throw failure( + "connection_failed", + "SSH tunnel did not become ready. Verify the desktop broker, grant expiry, and local port availability.", + ); + await writeCredential(options.output, local); + written = true; + if (exited) + throw failure("connection_lost", "SSH tunnel closed during setup."); + return { endpoint: local.endpoint, close, done }; + } catch (error) { + await close(); + throw error; + } +} diff --git a/src/desktop/server.ts b/src/desktop/server.ts index 9f17c34..4deb0ac 100644 --- a/src/desktop/server.ts +++ b/src/desktop/server.ts @@ -1,29 +1,31 @@ -import { createServer as httpServer } from "node:http"; -import { createServer as socketServer } from "node:net"; -import { randomBytes, createHash, timingSafeEqual } from "node:crypto"; +import { createHash, randomBytes, timingSafeEqual } from "node:crypto"; +import { createReadStream } from "node:fs"; import { - mkdir, - writeFile, chmod, - stat, + lstat, + mkdir, readFile, + stat, unlink, - lstat, + writeFile, } from "node:fs/promises"; -import { createReadStream } from "node:fs"; -import { dirname, join } from "node:path"; +import { createServer as httpServer, type Server } from "node:http"; +import { createServer as socketServer } from "node:net"; import { homedir } from "node:os"; +import { dirname, join } from "node:path"; import { z } from "zod"; import { X11Backend } from "./backend.js"; import { DesktopController } from "./controller.js"; -import { Target, Request, scopes, errorJSON, failure } from "./protocol.js"; +import { MacBackend, type MacDriver } from "./mac-backend.js"; +import { connectMacDriver } from "./mac-driver.js"; +import { errorJSON, failure, Request, scopes, Target } from "./protocol.js"; import { desktopHTML } from "./view.js"; export const Config = z.object({ - display: z.string().regex(/^(?:unix)?:(\d+)(?:\.(\d+))?$/), + display: z.string().regex(/^(?:macos|(?:unix)?:(\d+)(?:\.(\d+))?)$/), authority: z.string().optional(), uid: z.number().int().nonnegative(), generation: z.string().min(1).max(128), - helper: z.string(), + helper: z.string().default(""), directory: z.string().optional(), origin: z.string().url().optional(), basePath: z @@ -32,25 +34,25 @@ export const Config = z.object({ .default(""), }); export type Config = z.infer; -export { registryPath } from "../linux/display-registry.js"; -import { - canonical, - displayKey, - registryPath, -} from "../linux/display-registry.js"; +export { registryPath } from "./registry.js"; + +import { canonical, displayKey, registryPath } from "./registry.js"; export async function startDesktop(input: z.input) { const config = Config.parse(input); - if (process.platform !== "linux") + const mac = config.display === "macos"; + if (mac ? process.platform !== "darwin" : process.platform !== "linux") throw failure( "unsupported", - "Display service currently requires Linux X11.", + "Use display macos on macOS, or a local X11 display on Linux.", ); + if (!mac && !config.helper) + throw failure("invalid_request", "Linux requires an X11 helper path."); if (config.uid !== process.getuid?.()) throw failure( "permission_denied", "Configured display user differs from the service user.", ); - if (config.authority) + if (!mac && config.authority) await stat(config.authority).catch(() => { throw failure("permission_denied", "Xauthority file is unavailable."); }); @@ -66,288 +68,342 @@ export async function startDesktop(input: z.input) { ), ), ); - lock.listen( - "\0opcode-desktop-" + - createHash("sha256").update(display).digest("hex").slice(0, 24), - resolve, - ); - }); - await mkdir(dirname(path), { recursive: true, mode: 0o700 }); - const directoryInfo = await lstat(dirname(path)); - if ( - !directoryInfo.isDirectory() || - directoryInfo.isSymbolicLink() || - directoryInfo.uid !== process.getuid?.() || - directoryInfo.mode & 0o077 - ) { - lock.close(); - throw failure("permission_denied", "Unsafe broker directory."); - } - const directory = - config.directory ?? - join(homedir(), ".local/state/opcode/recordings", displayKey(display)); - const backend = new X11Backend(config.helper, { - ...process.env, - DISPLAY: display, - ...(config.authority ? { XAUTHORITY: config.authority } : {}), + // A loopback reservation avoids stale filesystem sockets after a Mac crash. + // Linux retains its namespace-wide abstract X11 display reservation. + if (mac) lock.listen(49152 + (config.uid % 16000), "127.0.0.1", resolve); + else + lock.listen( + "\0opcode-desktop-" + + createHash("sha256").update(display).digest("hex").slice(0, 24), + resolve, + ); }); - const controller = new DesktopController( - backend, - config.generation, - directory, - ); - let closed = false; + let startupController: DesktopController | undefined; + let startupDriver: MacDriver | undefined; + let startupServer: Server | undefined; try { - await controller.init(); - } catch (e) { - lock.close(); - await controller.close(); - throw e; - } - const master = randomBytes(32).toString("hex"); - let host = controller.authority.issue("host", [...scopes], 3600000); - let inFlight = 0; - const base = config.basePath; - const authenticate = ( - header: string | undefined, - scope?: (typeof scopes)[number], - ) => { - const token = header?.replace(/^Bearer /, "") ?? ""; - const admin = - token.length === master.length && - timingSafeEqual(Buffer.from(token), Buffer.from(master)); - if (admin) { - try { - controller.authority.verify(host.token); - } catch { - host = controller.authority.issue("host", [...scopes], 3600000); - } - return { grant: controller.authority.verify(host.token, scope), admin }; + await mkdir(dirname(path), { recursive: true, mode: 0o700 }); + const directoryInfo = await lstat(dirname(path)); + if ( + !directoryInfo.isDirectory() || + directoryInfo.isSymbolicLink() || + directoryInfo.uid !== process.getuid?.() || + directoryInfo.mode & 0o077 + ) { + lock.close(); + throw failure("permission_denied", "Unsafe broker directory."); } - return { grant: controller.authority.verify(token, scope), admin: false }; - }; - const server = httpServer(async (req, res) => { - res.setHeader("Cache-Control", "no-store"); - res.setHeader("Referrer-Policy", "no-referrer"); - res.setHeader("X-Content-Type-Options", "nosniff"); - const send = (status: number, value: unknown) => { - res.writeHead(status, { "Content-Type": "application/json" }); - res.end(JSON.stringify(value)); - }; - let counted = false; - const finished = () => { - if (counted) { - counted = false; - inFlight--; - } - }; - res.once("finish", finished); - res.once("close", finished); - try { - const expected = config.origin ?? origin; - if (req.headers.origin && req.headers.origin !== expected) - throw failure("permission_denied", "Origin is not allowed."); - if (req.headers.origin === expected) { - res.setHeader("Access-Control-Allow-Origin", expected); - res.setHeader("Vary", "Origin"); - } - const url = new URL(req.url ?? "/", origin); - const route = url.pathname; - if (route === base + "/view" && req.method === "GET") { - res.setHeader("Content-Type", "text/html; charset=utf-8"); - res.setHeader( - "Content-Security-Policy", - "default-src 'none'; script-src 'unsafe-inline'; style-src 'unsafe-inline'; img-src data:; connect-src 'self'; frame-ancestors " + - (config.origin ?? "'self'"), + const directory = + config.directory ?? + join(homedir(), ".local/state/opcode/recordings", displayKey(display)); + const macDriver = mac ? await connectMacDriver() : undefined; + startupDriver = macDriver; + if (macDriver) { + const status = z + .object({ accessibility: z.boolean(), screenRecording: z.boolean() }) + .parse(await macDriver.request("status")); + if (!status.accessibility || !status.screenRecording) + throw failure( + "permission_denied", + "Enable Accessibility and Screen Recording for the installed Opcode helper, then attach again.", ); - res.end(desktopHTML(base)); - return; - } - if (req.method === "OPTIONS") { - res.writeHead(204, { - "Access-Control-Allow-Headers": "Authorization, Content-Type", - "Access-Control-Allow-Methods": "GET, POST, DELETE, OPTIONS", - }); - res.end(); - return; - } - const { grant, admin } = authenticate(req.headers.authorization); - if (route === base + "/session") { - send(200, { - generation: controller.generation, - protocol: 1, - scopes: grant.scopes, + } + const backend = macDriver + ? new MacBackend(macDriver) + : new X11Backend(config.helper, { + ...process.env, + DISPLAY: display, + ...(config.authority ? { XAUTHORITY: config.authority } : {}), }); - return; + const controller = new DesktopController( + backend, + config.generation, + directory, + ); + startupController = controller; + let closed = false; + await controller.init(); + const master = randomBytes(32).toString("hex"); + let host = controller.authority.issue("host", [...scopes], 3600000); + let inFlight = 0; + const base = config.basePath; + const authenticate = ( + header: string | undefined, + scope?: (typeof scopes)[number], + ) => { + const token = header?.replace(/^Bearer /, "") ?? ""; + const admin = + token.length === master.length && + timingSafeEqual(Buffer.from(token), Buffer.from(master)); + if (admin) { + try { + controller.authority.verify(host.token); + } catch { + host = controller.authority.issue("host", [...scopes], 3600000); + } + return { grant: controller.authority.verify(host.token, scope), admin }; } - if (route === base + "/frame" && req.method === "GET") { - authenticate(req.headers.authorization, "viewer-read"); - if (inFlight >= 32) - throw failure("overloaded", "Too many concurrent requests."); + return { grant: controller.authority.verify(token, scope), admin: false }; + }; + const server = httpServer(async (req, res) => { + res.setHeader("Cache-Control", "no-store"); + res.setHeader("Referrer-Policy", "no-referrer"); + res.setHeader("X-Content-Type-Options", "nosniff"); + const send = (status: number, value: unknown) => { + res.writeHead(status, { "Content-Type": "application/json" }); + res.end(JSON.stringify(value)); + }; + let counted = false; + const finished = () => { + if (counted) { + counted = false; + inFlight--; + } + }; + res.once("finish", finished); + res.once("close", finished); + try { + const expected = req.headers.origin; + const allowed = + !expected || expected === origin || expected === config.origin; + if (!allowed) + throw failure("permission_denied", "Origin is not allowed."); + if (expected && allowed) { + res.setHeader("Access-Control-Allow-Origin", expected); + res.setHeader("Vary", "Origin"); + } + const url = new URL(req.url ?? "/", origin); + const route = url.pathname; + if (route === base + "/view" && req.method === "GET") { + res.setHeader("Content-Type", "text/html; charset=utf-8"); + res.setHeader( + "Content-Security-Policy", + "default-src 'none'; script-src 'unsafe-inline'; style-src 'unsafe-inline'; img-src data:; connect-src 'self'; frame-ancestors " + + (config.origin ?? "'self'"), + ); + res.end(desktopHTML(base)); + return; + } + if (req.method === "OPTIONS") { + res.writeHead(204, { + "Access-Control-Allow-Headers": "Authorization, Content-Type", + "Access-Control-Allow-Methods": "GET, POST, DELETE, OPTIONS", + }); + res.end(); + return; + } + const { grant, admin } = authenticate(req.headers.authorization); + if (route === base + "/session") { + send(200, { + generation: controller.generation, + protocol: 1, + scopes: grant.scopes, + admin, + subject: grant.subject, + }); + return; + } + if (route === base + "/frame" && req.method === "GET") { + authenticate(req.headers.authorization, "viewer-read"); + if (inFlight >= 32) + throw failure("overloaded", "Too many concurrent requests."); + inFlight++; + counted = true; + const windowId = url.searchParams.get("windowId"); + const target = Target.parse( + windowId === null + ? { kind: "display" } + : { kind: "window", id: Number(windowId) }, + ); + const frame = await controller.observe(grant, target); + authenticate(req.headers.authorization, "viewer-read"); + send(200, { + ...frame, + ...controller.multiplayerState(), + recording: controller.recorder + .list() + .filter((r) => r.state === "recording" || r.state === "paused") + .map(({ id, state }) => ({ id, state })), + }); + return; + } + if (route.startsWith(base + "/artifacts/") && req.method === "GET") { + authenticate(req.headers.authorization, "recording-read"); + const id = route.slice((base + "/artifacts/").length); + if (!/^[a-f0-9-]{36}$/.test(id)) + throw failure("not_found", "Artifact not found."); + const file = controller.recorder.path(id), + info = await stat(file); + let start = 0, + end = info.size - 1; + if (req.headers.range) { + const match = /^bytes=(\d+)-(\d*)$/.exec(req.headers.range); + if (!match) + throw failure("invalid_range", "Use one explicit byte range."); + start = +match[1]; + end = match[2] ? +match[2] : end; + if (start > end || end >= info.size) { + res.writeHead(416, { "Content-Range": `bytes */${info.size}` }); + res.end(); + return; + } + } + res.writeHead(req.headers.range ? 206 : 200, { + "Content-Type": "video/mp4", + "Accept-Ranges": "bytes", + "Content-Length": end - start + 1, + "Content-Disposition": `attachment; filename="${id}.mp4"`, + ...(req.headers.range + ? { "Content-Range": `bytes ${start}-${end}/${info.size}` } + : {}), + }); + const stream = createReadStream(file, { start, end }); + const timer = setInterval(() => { + try { + authenticate(req.headers.authorization, "recording-read"); + } catch { + stream.destroy(); + res.destroy(); + } + }, 100); + stream.on("error", () => res.destroy()); + res.on("close", () => { + clearInterval(timer); + stream.destroy(); + }); + stream.pipe(res); + return; + } + if (route !== base + "/rpc" || req.method !== "POST") + throw failure("not_found", "Route not found."); + let body = ""; + for await (const chunk of req) { + body += chunk.toString(); + if (Buffer.byteLength(body) > 128000) + throw failure("invalid_request", "Request exceeds 128 KB."); + } + const request = Request.parse(JSON.parse(body)); + if (request.method === "shutdown") { + if (!admin) + throw failure("permission_denied", "Host authority required."); + if (request.generation !== controller.generation) + throw failure("generation_mismatch", "Lifecycle mismatch."); + send(200, { + ok: true, + protocol: 1, + generation: controller.generation, + result: { stopping: true }, + }); + setImmediate(() => void close()); + return; + } + const priority = [ + "stop", + "takeover", + "release", + "rebind", + "revoke", + ].includes(request.method); + if (inFlight >= 32 && !priority) + throw failure("overloaded", "Request limit reached."); inFlight++; counted = true; - const windowId = url.searchParams.get("windowId"); - const target = Target.parse( - windowId === null - ? { kind: "display" } - : { kind: "window", id: Number(windowId) }, - ); - const frame = await controller.observe(grant, target); - authenticate(req.headers.authorization, "viewer-read"); - send(200, { - ...frame, - recording: controller.recorder - .list() - .filter((r) => r.state === "recording" || r.state === "paused") - .map(({ id, state }) => ({ id, state })), + res.once("close", () => { + if (!res.writableEnded && request.method === "input") + void controller.cancelGrantInput(grant.id).catch(() => {}); }); - return; - } - if (route.startsWith(base + "/artifacts/") && req.method === "GET") { - authenticate(req.headers.authorization, "recording-read"); - const id = route.slice((base + "/artifacts/").length); - if (!/^[a-f0-9-]{36}$/.test(id)) - throw failure("not_found", "Artifact not found."); - const file = controller.recorder.path(id), - info = await stat(file); - let start = 0, - end = info.size - 1; - if (req.headers.range) { - const match = /^bytes=(\d+)-(\d*)$/.exec(req.headers.range); - if (!match) - throw failure("invalid_range", "Use one explicit byte range."); - start = +match[1]; - end = match[2] ? +match[2] : end; - if (start > end || end >= info.size) { - res.writeHead(416, { "Content-Range": `bytes */${info.size}` }); - res.end(); - return; - } + const result = await controller.call(request, grant, admin); + if ( + request.method !== "rebind" && + request.generation !== controller.generation + ) + throw failure( + "generation_mismatch", + "Lifecycle changed during the operation.", + [ + "health", + "state", + "apps", + "windows", + "observe", + "presence.list", + "recording.list", + ].includes(request.method) + ? "notDispatched" + : "unknown", + ); + if (request.method === "rebind") { + descriptor.generation = controller.generation; + await writeFile(path, JSON.stringify(descriptor), { mode: 0o600 }); } - res.writeHead(req.headers.range ? 206 : 200, { - "Content-Type": "video/mp4", - "Accept-Ranges": "bytes", - "Content-Length": end - start + 1, - "Content-Disposition": `attachment; filename="${id}.mp4"`, - ...(req.headers.range - ? { "Content-Range": `bytes ${start}-${end}/${info.size}` } - : {}), - }); - const stream = createReadStream(file, { start, end }); - const timer = setInterval(() => { - try { - authenticate(req.headers.authorization, "recording-read"); - } catch { - stream.destroy(); - res.destroy(); - } - }, 100); - stream.on("error", () => res.destroy()); - res.on("close", () => { - clearInterval(timer); - stream.destroy(); + send(200, { + ok: true, + protocol: 1, + generation: controller.generation, + result, }); - stream.pipe(res); - return; - } - if (route !== base + "/rpc" || req.method !== "POST") - throw failure("not_found", "Route not found."); - let body = ""; - for await (const chunk of req) { - body += chunk.toString(); - if (Buffer.byteLength(body) > 128000) - throw failure("invalid_request", "Request exceeds 128 KB."); - } - const request = Request.parse(JSON.parse(body)); - if (request.method === "shutdown") { - if (!admin) - throw failure("permission_denied", "Host authority required."); - if (request.generation !== controller.generation) - throw failure("generation_mismatch", "Lifecycle mismatch."); - send(200, { ok: true, result: { stopping: true } }); - setImmediate(() => void close()); - return; - } - const priority = [ - "stop", - "takeover", - "release", - "rebind", - "revoke", - ].includes(request.method); - if (inFlight >= 32 && !priority) - throw failure("overloaded", "Request limit reached."); - inFlight++; - counted = true; - res.once("close", () => { - if (!res.writableEnded && request.method === "input") - void controller.cancelGrantInput(grant.id).catch(() => {}); - }); - const result = await controller.call(request, grant, admin); - if ( - request.method !== "rebind" && - request.generation !== controller.generation - ) - throw failure( - "generation_mismatch", - "Lifecycle changed during the operation.", - ); - if (request.method === "rebind") { - descriptor.generation = controller.generation; - await writeFile(path, JSON.stringify(descriptor), { mode: 0o600 }); + } catch (e) { + if (!res.headersSent) + send( + (e as { code?: string }).code === "permission_denied" ? 403 : 400, + { ok: false, error: errorJSON(e) }, + ); + else res.destroy(); + } finally { + if (res.writableFinished || res.destroyed) finished(); } - send(200, { - ok: true, - protocol: 1, - generation: controller.generation, - result, - }); - } catch (e) { - if (!res.headersSent) - send( - (e as { code?: string }).code === "permission_denied" ? 403 : 400, - { ok: false, error: errorJSON(e) }, - ); - else res.destroy(); - } finally { - if (res.writableFinished || res.destroyed) finished(); - } - }); - server.maxConnections = 64; - server.maxRequestsPerSocket = 100; - server.requestTimeout = 15000; - server.headersTimeout = 10000; - await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); - const address = server.address(); - if (!address || typeof address === "string") throw Error("No listener"); - const origin = `http://127.0.0.1:${address.port}`; - const descriptor = { - protocol: 1, - display, - generation: controller.generation, - endpoint: origin + base, - token: master, - pid: process.pid, - }; - await mkdir(dirname(path), { recursive: true, mode: 0o700 }); - const dirInfo = await lstat(dirname(path)); - if ( - !dirInfo.isDirectory() || - dirInfo.isSymbolicLink() || - dirInfo.uid !== process.getuid?.() || - dirInfo.mode & 0o077 - ) - throw failure("permission_denied", "Unsafe broker directory."); - await writeFile(path, JSON.stringify(descriptor), { mode: 0o600 }); - await chmod(path, 0o600); - const close = async () => { - if (closed) return; - closed = true; - await controller.close(); - server.closeAllConnections(); - await new Promise((resolve) => server.close(() => resolve())); - await unlink(path).catch(() => {}); + }); + startupServer = server; + server.maxConnections = 64; + + server.requestTimeout = 15000; + server.headersTimeout = 10000; + await new Promise((resolve, reject) => { + server.once("error", reject); + server.listen(0, "127.0.0.1", resolve); + }); + const address = server.address(); + if (!address || typeof address === "string") throw Error("No listener"); + const origin = `http://127.0.0.1:${address.port}`; + const descriptor = { + protocol: 1, + display, + generation: controller.generation, + endpoint: origin + base, + token: master, + pid: process.pid, + }; + await mkdir(dirname(path), { recursive: true, mode: 0o700 }); + const dirInfo = await lstat(dirname(path)); + if ( + !dirInfo.isDirectory() || + dirInfo.isSymbolicLink() || + dirInfo.uid !== process.getuid?.() || + dirInfo.mode & 0o077 + ) + throw failure("permission_denied", "Unsafe broker directory."); + await writeFile(path, JSON.stringify(descriptor), { mode: 0o600 }); + await chmod(path, 0o600); + const close = async () => { + if (closed) return; + closed = true; + await controller.close(); + macDriver?.close?.(); + server.closeAllConnections(); + await new Promise((resolve) => server.close(() => resolve())); + await unlink(path).catch(() => {}); + lock.close(); + }; + return { controller, endpoint: descriptor.endpoint, path, close }; + } catch (error) { + await startupController?.close().catch(() => {}); + startupDriver?.close?.(); + startupServer?.closeAllConnections(); + if (startupServer?.listening) + await new Promise((resolve) => + startupServer!.close(() => resolve()), + ); lock.close(); - }; - return { controller, endpoint: descriptor.endpoint, path, close }; + throw error; + } } diff --git a/src/desktop/view.ts b/src/desktop/view.ts index b738a3f..c7a5f9c 100644 --- a/src/desktop/view.ts +++ b/src/desktop/view.ts @@ -1,25 +1,34 @@ export function desktopHTML(base: string) { - return `Opcode desktop
OpcodeConnecting…
Desktop stream`; } diff --git a/tests/desktop-client.test.ts b/tests/desktop-client.test.ts new file mode 100644 index 0000000..7431ea8 --- /dev/null +++ b/tests/desktop-client.test.ts @@ -0,0 +1,96 @@ +import assert from "node:assert/strict"; +import { chmod, mkdtemp, readFile, rm, symlink } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { test } from "node:test"; +import { + desktopCall, + readCredential, + serviceURL, + writeCredential, +} from "../src/desktop/client.js"; + +const credential = { + endpoint: "https://mac.example.test/desktop", + token: "a".repeat(64), + generation: "boot-one", +}; +test("remote endpoints require TLS; loopback tunnels are allowed without URL credentials", () => { + for (const value of [ + credential.endpoint, + "http://127.0.0.1:4311", + "http://[::1]:4311/desktop", + ]) + assert.ok(serviceURL(value)); + for (const value of [ + "http://mac.example.test", + "http://100.64.0.1", + "https://user:secret@mac.example.test", + "https://mac.example.test?token=x", + "https://mac.example.test#secret", + "file:///tmp/socket", + "http://127.0.0.1.evil.test", + ]) + assert.throws(() => serviceURL(value), /HTTPS/); +}); +test("credentials are private, never overwritten, and symlinks are rejected", async () => { + const dir = await mkdtemp(join(tmpdir(), "cu-credentials-")); + try { + const path = join(dir, "user.json"); + await writeCredential(path, credential); + assert.deepEqual(await readCredential(path), credential); + await assert.rejects( + writeCredential(path, { ...credential, token: "b".repeat(64) }), + ); + assert.equal( + JSON.parse(await readFile(path, "utf8")).token, + credential.token, + ); + await symlink(path, join(dir, "link.json")); + await assert.rejects(readCredential(join(dir, "link.json"))); + await chmod(path, 0o644); + await assert.rejects(readCredential(path), /private/); + } finally { + await rm(dir, { recursive: true, force: true }); + } +}); +test("remote calls bind generation, disable redirects and never retry uncertain input", async () => { + let count = 0; + const transport = (async ( + url: string | URL | Request, + init?: RequestInit, + ) => { + count++; + assert.ok(init); + assert.equal(url, credential.endpoint + "/rpc"); + assert.equal(init?.redirect, "error"); + assert.equal( + (init.headers as Record).Authorization, + "Bearer " + credential.token, + ); + assert.equal(JSON.parse(init?.body as string).generation, "boot-one"); + throw new Error("socket lost"); + }) as typeof fetch; + await assert.rejects( + desktopCall(credential, "input", {}, { fetch: transport }), + (error: any) => + error.delivery === "unknown" && error.code === "connection_lost", + ); + assert.equal(count, 1); + const result = (async () => + Response.json({ + ok: true, + protocol: 1, + generation: "boot-two", + result: {}, + })) as typeof fetch; + await assert.rejects( + desktopCall(credential, "state", {}, { fetch: result }), + /lifecycle changed/, + ); + const wrong = (async () => Response.json(null)) as typeof fetch; + await assert.rejects( + desktopCall(credential, "input", {}, { fetch: wrong }), + (error: any) => error.delivery === "unknown", + ); +}); diff --git a/tests/desktop-controller.test.ts b/tests/desktop-controller.test.ts index 45ab3da..4a507d5 100644 --- a/tests/desktop-controller.test.ts +++ b/tests/desktop-controller.test.ts @@ -1,24 +1,26 @@ -import { test } from "node:test"; import assert from "node:assert/strict"; -import { mkdtemp, rm } from "node:fs/promises"; +import { randomUUID } from "node:crypto"; +import { mkdtemp, readFile, rm } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; -import { randomUUID } from "node:crypto"; -import { DesktopController } from "../src/desktop/controller.js"; -import { X11Backend } from "../src/desktop/backend.js"; +import { test } from "node:test"; import { Authority } from "../src/desktop/authority.js"; +import { X11Backend } from "../src/desktop/backend.js"; +import { DesktopController } from "../src/desktop/controller.js"; import { - scopes, - type Request, type Action, + type Request, + scopes, type Target, } from "../src/desktop/protocol.js"; + class Fixture extends X11Backend { delivered: Action[] = []; block = false; color = 0; width = 640; releaseCount = 0; + failRelease = false; override async geometry() { return { id: 1, x: 0, y: 0, width: this.width, height: 480 }; } @@ -35,6 +37,7 @@ class Fixture extends X11Backend { } override async input(_t: Target, a: Action, s: AbortSignal) { if (a.kind === "keyUp") { + if (this.failRelease) throw Error("driver disconnected"); this.releaseCount++; return; } @@ -82,7 +85,11 @@ test("one controller, takeover cancels queued input and releases held keys; fres }), a, ); - const failure = assert.rejects(action, /may have been delivered/); + const failure = assert.rejects(action, (error: unknown) => { + assert.equal((error as { delivery: string }).delivery, "unknown"); + assert.equal((error as { code: string }).code, "outcome_unknown"); + return true; + }); await new Promise((r) => setTimeout(r, 10)); const queued = c.call( request("input", { @@ -152,3 +159,185 @@ test("duplicate mutations do not replay; changed pixels, geometry and generation await rm(directory, { recursive: true, force: true }); } }); + +test("multiplayer cursors require presence scope and do not grant OS input; participant leave fences held input", async () => { + const directory = await mkdtemp(join(tmpdir(), "opcode-multiplayer-")); + const backend = new Fixture("", {}); + const c = new DesktopController(backend, "one", directory); + await c.init(); + try { + const viewer = c.authority.issue("viewer", ["viewer-read"], 60000); + const participant = c.authority.issue( + "alice", + ["viewer-read", "presence"], + 60000, + ); + const controller = c.authority.issue("bob", [...scopes], 60000); + await assert.rejects(c.call(request("presence.join"), viewer), /scope/); + const alice = (await c.call( + request("presence.join", { name: "Alice", role: "human" }), + participant, + )) as { id: string }; + await assert.rejects( + c.call(request("takeover", { participantId: alice.id }), participant), + /scope/, + ); + await assert.rejects( + c.call(request("takeover", { participantId: alice.id }), controller), + /another credential/, + ); + const bob = (await c.call( + request("presence.join", { name: "Bob" }), + controller, + )) as { id: string }; + const lease = (await c.call( + request("takeover", { participantId: bob.id }), + controller, + )) as { id: string }; + const observation = await c.observe(controller, { kind: "display" }); + await c.call( + request("input", { + leaseId: lease.id, + observationId: observation.observationId, + action: { kind: "keyDown", key: "Shift" }, + }), + controller, + ); + await c.call( + request("presence.leave", { participantId: alice.id }), + participant, + ); + assert.equal(c.multiplayerState().controller?.participantId, bob.id); + await c.call( + request("presence.leave", { participantId: bob.id }), + controller, + ); + assert.equal(c.multiplayerState().controller, null); + assert.equal(backend.releaseCount, 1); + await assert.rejects( + c.call(request("renew", { leaseId: lease.id }), controller), + /Acquire/, + ); + c.authority.revoke(controller.id); + await assert.rejects( + c.call(request("presence.join"), controller), + /revoked/, + ); + } finally { + await c.close(); + await rm(directory, { recursive: true, force: true }); + } +}); + +test("revocation and rebind remove multiplayer identities and invalidate participant-bound leases", async () => { + const directory = await mkdtemp(join(tmpdir(), "opcode-multiplayer-")); + const c = new DesktopController(new Fixture("", {}), "one", directory); + await c.init(); + try { + const host = c.authority.issue("host", [...scopes], 60000); + const guest = c.authority.issue("guest", [...scopes], 60000); + const participant = (await c.call(request("presence.join"), guest)) as { + id: string; + }; + await c.call(request("takeover", { participantId: participant.id }), guest); + await c.call(request("revoke", { id: guest.id }), host, true); + assert.deepEqual(c.multiplayerState(), { + participants: [], + controller: null, + }); + await c.call(request("presence.join"), host); + await c.call(request("rebind", { generation: "two" }), host, true); + assert.deepEqual(c.multiplayerState(), { + participants: [], + controller: null, + }); + await assert.rejects(c.call(request("presence.join"), host), /rebind/); + } finally { + await c.close(); + await rm(directory, { recursive: true, force: true }); + } +}); + +test("a busy observer cannot evict another participant's actionable observation", async () => { + const directory = await mkdtemp(join(tmpdir(), "opcode-multiplayer-")); + const backend = new Fixture("", {}); + const c = new DesktopController(backend, "one", directory); + await c.init(); + try { + const agent = c.authority.issue("agent", [...scopes], 60000); + const viewer = c.authority.issue("viewer", ["viewer-read"], 60000); + const lease = (await c.call(request("acquire"), agent)) as { id: string }; + const observation = await c.observe(agent, { kind: "display" }); + for (let i = 0; i < 40; i++) await c.observe(viewer, { kind: "display" }); + await c.call( + request("input", { + leaseId: lease.id, + observationId: observation.observationId, + action: { kind: "click", x: 1, y: 1 }, + }), + agent, + ); + assert.equal(backend.delivered.length, 1); + } finally { + await c.close(); + await rm(directory, { recursive: true, force: true }); + } +}); + +test("failed key release remains journaled and blocks acquisition until restart cleanup succeeds", async () => { + const directory = await mkdtemp(join(tmpdir(), "opcode-recovery-")); + const backend = new Fixture("", {}); + const controller = new DesktopController(backend, "one", directory); + await controller.init(); + try { + const grant = controller.authority.issue("agent", [...scopes], 60000); + const lease = (await controller.call(request("acquire"), grant)) as { + id: string; + }; + const observation = await controller.observe(grant, { kind: "display" }); + await controller.call( + request("input", { + leaseId: lease.id, + observationId: observation.observationId, + action: { kind: "keyDown", key: "Shift" }, + }), + grant, + ); + backend.failRelease = true; + await assert.rejects( + controller.call(request("stop"), grant), + /could not be released/, + ); + assert.deepEqual( + JSON.parse(await readFile(join(directory, "held-input.json"), "utf8")) + .keys, + ["Shift"], + ); + await assert.rejects( + controller.call(request("takeover"), grant), + /transition/, + ); + await controller.close(); + assert.deepEqual( + JSON.parse(await readFile(join(directory, "held-input.json"), "utf8")) + .keys, + ["Shift"], + ); + const recoveredBackend = new Fixture("", {}); + const recovered = new DesktopController(recoveredBackend, "two", directory); + try { + await recovered.init(); + assert.equal(recoveredBackend.releaseCount, 1); + assert.deepEqual( + JSON.parse(await readFile(join(directory, "held-input.json"), "utf8")) + .keys, + [], + ); + } finally { + await recovered.close(); + } + } finally { + await controller.close(); + await rm(directory, { recursive: true, force: true }); + } +}); diff --git a/tests/desktop-http.test.ts b/tests/desktop-http.test.ts new file mode 100644 index 0000000..72a8726 --- /dev/null +++ b/tests/desktop-http.test.ts @@ -0,0 +1,142 @@ +import assert from "node:assert/strict"; +import { randomUUID } from "node:crypto"; +import { chmod, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { test } from "node:test"; +import { desktopCall } from "../src/desktop/client.js"; +import { startDesktop } from "../src/desktop/server.js"; + +// Exercise the real authenticated HTTP server without requiring an X server. +test("HTTP multiplayer isolates grants, publishes cursor state, and fences revoke/rebind", { + skip: process.platform !== "linux", +}, async () => { + const directory = await mkdtemp(join(tmpdir(), "opcode-http-")); + const helper = join(directory, "helper.cjs"); + await writeFile( + helper, + `#!/usr/bin/env node +const op=process.argv[2]; +if(op==='display') console.log(JSON.stringify({id:1,x:0,y:0,width:2,height:2})); +else if(op==='state') console.log(JSON.stringify({focus:1,x:0,y:0})); +else if(op==='capture') process.stdout.write(Buffer.concat([Buffer.from('P6\\n2 2\\n255\\n'),Buffer.alloc(12)])); +else if(op==='list') console.log('[]'); +else console.log('{}'); +`, + ); + await chmod(helper, 0o700); + let service: Awaited> | undefined; + try { + service = await startDesktop({ + display: `:${100000 + Math.floor(Math.random() * 100000000)}`, + uid: process.getuid!(), + generation: "one", + helper, + directory: join(directory, "data"), + }); + const endpoint = service.endpoint; + const host = JSON.parse(await readFile(service.path, "utf8")) + .token as string; + const rpc = async ( + token: string, + method: string, + args: Record = {}, + generation = "one", + ) => { + const response = await fetch(endpoint + "/rpc", { + method: "POST", + headers: { + Authorization: `Bearer ${token}`, + "Content-Type": "application/json", + }, + body: JSON.stringify({ id: randomUUID(), generation, method, args }), + }); + return { + status: response.status, + ...((await response.json()) as { + ok: boolean; + result: any; + error?: { code: string }; + }), + }; + }; + const alice = ( + await rpc(host, "grant", { + subject: "alice", + scopes: ["viewer-read", "presence"], + ttlMs: 60000, + }) + ).result; + const bob = ( + await rpc(host, "grant", { + subject: "bob", + scopes: ["viewer-read", "presence", "input-control"], + ttlMs: 60000, + }) + ).result; + const anonymous = await fetch(endpoint + "/frame"); + assert.equal(anonymous.status, 403); + const ap = (await rpc(alice.token, "presence.join", { name: "Alice" })) + .result; + const bp = (await rpc(bob.token, "presence.join", { name: "Bob" })).result; + assert.equal( + (await rpc(alice.token, "takeover", { participantId: ap.id })).status, + 403, + ); + assert.equal( + ( + await rpc(bob.token, "presence.update", { + participantId: ap.id, + cursor: { x: 0.2, y: 0.3 }, + }) + ).ok, + false, + ); + assert.equal( + ( + await rpc(alice.token, "presence.update", { + participantId: ap.id, + cursor: { x: 0.2, y: 0.3 }, + }) + ).ok, + true, + ); + const lease = (await rpc(bob.token, "takeover", { participantId: bp.id })) + .result; + assert.ok(lease.id); + const frameResponse = await fetch(endpoint + "/frame", { + headers: { Authorization: `Bearer ${alice.token}` }, + }); + assert.equal(frameResponse.status, 200); + const frame = (await frameResponse.json()) as any; + assert.equal(frame.controller.participantId, bp.id); + assert.deepEqual( + frame.participants.find((p: any) => p.id === ap.id).cursor, + { x: 0.2, y: 0.3 }, + ); + assert.ok(frame.image.base64); + assert.equal((await rpc(alice.token, "state")).result.lease.subject, "bob"); + await rpc(host, "revoke", { id: bob.id }); + const state = (await rpc(alice.token, "presence.list")).result; + assert.equal(state.controller, null); + assert.equal(state.participants.length, 1); + assert.equal((await rpc(bob.token, "state")).status, 403); + await rpc(host, "rebind", { generation: "two" }); + assert.equal((await rpc(alice.token, "state", {}, "two")).status, 403); + assert.deepEqual((await rpc(host, "presence.list", {}, "two")).result, { + participants: [], + controller: null, + }); + assert.deepEqual( + await desktopCall( + { endpoint, token: host, generation: "two" }, + "shutdown", + {}, + ), + { stopping: true }, + ); + } finally { + await service?.close(); + await rm(directory, { recursive: true, force: true }); + } +}); diff --git a/tests/desktop-platform.test.ts b/tests/desktop-platform.test.ts new file mode 100644 index 0000000..90d2085 --- /dev/null +++ b/tests/desktop-platform.test.ts @@ -0,0 +1,36 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { + canonical, + displayKey, + registryPath, +} from "../src/desktop/registry.js"; +import { Config, startDesktop } from "../src/desktop/server.js"; +import { registryPath as x11Path } from "../src/linux/display-registry.js"; + +test("Mac registry is stable and separate from X11 display identities", () => { + assert.equal(canonical("macos"), "macos"); + assert.equal(canonical("unix:01"), ":1.0"); + assert.equal(registryPath(":1"), x11Path(":1")); + assert.equal(registryPath("macos"), registryPath("macos")); + assert.notEqual(displayKey("macos"), displayKey(":0")); + assert.throws(() => canonical("remote:0")); +}); +test("Mac configuration needs no X11 helper and rejects arbitrary displays", () => { + assert.equal( + Config.parse({ display: "macos", uid: 501, generation: "test" }).helper, + "", + ); + assert.equal( + Config.safeParse({ display: "evil-host:0", uid: 501, generation: "test" }) + .success, + false, + ); +}); +test("Unsupported platform fails before allocating a broker", async () => { + const display = process.platform === "darwin" ? ":99" : "macos"; + await assert.rejects( + startDesktop({ display, uid: process.getuid?.() ?? 0, generation: "test" }), + (error: { code?: string }) => error.code === "unsupported", + ); +}); diff --git a/tests/desktop-presence.test.ts b/tests/desktop-presence.test.ts new file mode 100644 index 0000000..fc4a6a9 --- /dev/null +++ b/tests/desktop-presence.test.ts @@ -0,0 +1,57 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { Authority } from "../src/desktop/authority.js"; +import { Presence } from "../src/desktop/presence.js"; +import { methodArguments } from "../src/desktop/protocol.js"; + +test("participants are isolated by grant, bounded, and use visual-only normalized cursors", () => { + let now = 1000; + const authority = new Authority("one", () => now); + const presence = new Presence(authority, () => now); + const a = authority.issue("alice", ["presence"], 60000); + const b = authority.issue("bob", ["presence"], 60000); + const first = presence.join(a, "Alice", "human"); + const second = presence.join(a, "Other tab", "agent"); + assert.notEqual(first.id, second.id); + assert.throws( + () => presence.update(b, first.id, { x: 0.5, y: 0.5 }), + /another credential/, + ); + assert.throws(() => presence.leave(b, first.id), /another credential/); + assert.throws(() => + methodArguments["presence.update"].parse({ + participantId: first.id, + cursor: { x: 2, y: 0 }, + }), + ); + const update = presence.update(a, first.id, { x: 0.5, y: 0.25 }); + assert.deepEqual(update.cursor, { x: 0.5, y: 0.25 }); + assert.throws(() => presence.update(a, first.id, null), /limited/); + now += 100; + assert.deepEqual( + presence.update(a, first.id, undefined).cursor, + update.cursor, + ); + for (let i = 0; i < 6; i++) presence.join(a, `tab${i}`, "human"); + assert.throws(() => presence.join(a, "overflow", "human"), /limit/); + presence.leave(a, first.id); + assert.equal(presence.list().length, 7); +}); + +test("presence expires after disconnect and is removed on grant revocation and generation changes", () => { + let now = 1000; + const authority = new Authority("one", () => now); + const presence = new Presence(authority, () => now); + const grant = authority.issue("alice", ["presence"], 60000); + const p = presence.join(grant, undefined, "human"); + now += 15001; + assert.equal(presence.has(p.id), false); + assert.throws(() => presence.update(grant, p.id, null), /expired/); + presence.join(grant, undefined, "human"); + authority.revoke(grant.id); + assert.deepEqual(presence.list(), []); + const next = authority.issue("bob", ["presence"], 1000); + assert.equal(presence.join(next, undefined, "human").expiresAt, now + 1000); + authority.rebind("two"); + assert.deepEqual(presence.list(), []); +}); diff --git a/tests/desktop-remote.test.ts b/tests/desktop-remote.test.ts new file mode 100644 index 0000000..1a1b38f --- /dev/null +++ b/tests/desktop-remote.test.ts @@ -0,0 +1,298 @@ +import assert from "node:assert/strict"; +import { execFile } from "node:child_process"; +import { once } from "node:events"; +import { + chmod, + mkdtemp, + readFile, + rm, + stat, + writeFile, +} from "node:fs/promises"; +import { createServer as httpServer } from "node:http"; +import { createServer as netServer } from "node:net"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { test } from "node:test"; +import { promisify } from "node:util"; +import { readCredential } from "../src/desktop/client.js"; +import { + openSSHTunnel, + remoteCredentialCommand, + sshTarget, +} from "../src/desktop/remote.js"; + +const exec = promisify(execFile); + +const fakeSSH = `#!/usr/bin/env node +const net = require('node:net'); +const fs = require('node:fs'); +const args = process.argv.slice(2); +fs.appendFileSync(process.env.CU_TEST_SSH_LOG, JSON.stringify(args)+'\\n'); +if (!args.includes('-N')) { process.stdout.write(process.env.CU_TEST_REMOTE_CREDENTIAL); process.exit(0); } +const spec = args[args.indexOf('-L') + 1].split(':'); +const server = net.createServer(socket => { + const remote = net.connect(Number(spec[3]), spec[2]); + remote.on('error', () => socket.destroy()); socket.on('error', () => remote.destroy()); + socket.on('close', () => remote.destroy()); remote.on('close', () => socket.destroy()); + socket.pipe(remote); remote.pipe(socket); +}); +server.once('error', () => process.exit(1)); +server.listen(Number(spec[1]), spec[0], () => { + setTimeout(() => { + fs.writeFileSync(process.env.CU_TEST_SSH_MARKER, 'ready'); + process.stdout.write('OPCODE_TUNNEL_READY'); + }, 100); +}); +`; +async function freePort() { + const server = netServer(); + server.listen(0, "127.0.0.1"); + await once(server, "listening"); + const port = (server.address() as { port: number }).port; + await new Promise((resolve) => server.close(() => resolve())); + return port; +} +async function fixture(denied = false) { + const directory = await mkdtemp(join(tmpdir(), "cu-ssh-test-")); + const marker = join(directory, "ready"); + const log = join(directory, "ssh.log"); + await writeFile(join(directory, "ssh"), fakeSSH); + await chmod(join(directory, "ssh"), 0o700); + const token = "viewer-only-private-token"; + const requests: { method: string; authorization?: string; ready: boolean }[] = + []; + const server = httpServer(async (req, res) => { + let body = ""; + for await (const chunk of req) body += chunk; + const request = JSON.parse(body); + const ready = await stat(marker).then( + () => true, + () => false, + ); + requests.push({ + method: request.method, + authorization: req.headers.authorization, + ready, + }); + res.setHeader("Content-Type", "application/json"); + // This grant intentionally has no observe scope: readiness must use presence.list. + if ( + denied || + request.method !== "presence.list" || + req.headers.authorization !== `Bearer ${token}` + ) { + res.statusCode = 403; + res.end( + JSON.stringify({ + ok: false, + error: { code: "permission_denied", message: "Viewer scope only." }, + }), + ); + } else + res.end( + JSON.stringify({ + ok: true, + protocol: 1, + generation: "test-generation", + result: { participants: [] }, + }), + ); + }); + server.listen(0, "127.0.0.1"); + await once(server, "listening"); + const port = (server.address() as { port: number }).port; + const original = new Map(); + for (const [key, value] of Object.entries({ + PATH: directory + ":" + process.env.PATH, + CU_TEST_SSH_LOG: log, + CU_TEST_SSH_MARKER: marker, + CU_TEST_REMOTE_CREDENTIAL: JSON.stringify({ + endpoint: `http://127.0.0.1:${port}/desktop`, + token, + generation: "test-generation", + }), + })) { + original.set(key, process.env[key]); + process.env[key] = value; + } + return { + directory, + requests, + log, + token, + async close() { + for (const [key, value] of original) + if (value === undefined) delete process.env[key]; + else process.env[key] = value; + server.closeAllConnections(); + await new Promise((resolve) => server.close(() => resolve())); + await rm(directory, { recursive: true, force: true }); + }, + }; +} + +test("SSH target validation rejects option injection and shell metacharacters", () => { + for (const input of [ + "-oProxyCommand=evil", + "user@-host", + "host;touch", + "host\n", + "user@host:22", + "a b", + "$(evil)", + "", + ]) + assert.throws(() => sshTarget(input)); + for (const input of [ + "my-tailnet-mac", + "alice@mac.example.ts.net", + "my_alias", + ]) + assert.equal(sshTarget(input), input); +}); +test("Remote credential command treats malicious path content literally", async () => { + const directory = await mkdtemp(join(tmpdir(), "cu-path-test-")); + try { + const path = join( + directory, + "quote' $(printf exploited) `printf bad` ; $HOME.json", + ); + await writeFile(path, "literal-only"); + const { stdout } = await exec("/bin/sh", [ + "-c", + remoteCredentialCommand(path), + ]); + assert.equal(stdout, "literal-only"); + for (const input of [ + "relative.json", + "/tmp/new\nline", + "/tmp/null\0byte", + "/tmp/cr\rfile", + ]) + assert.throws(() => remoteCredentialCommand(input)); + } finally { + await rm(directory, { recursive: true, force: true }); + } +}); + +// Tests run sequentially because they temporarily select a fake ssh via PATH. +test("waits for forwarding readiness, supports viewer credentials, and deletes credentials on close", async () => { + const f = await fixture(); + let tunnel: Awaited> | undefined; + try { + const output = join(f.directory, "local.json"); + tunnel = await openSSHTunnel({ + host: "alice@mac", + remoteCredentialFile: "/Users/alice/viewer.json", + output, + port: await freePort(), + }); + assert.deepEqual(f.requests, [ + { + method: "presence.list", + authorization: `Bearer ${f.token}`, + ready: true, + }, + ]); + const credential = await readCredential(output); + assert.equal(credential.token, f.token); + assert.match(credential.endpoint, /^http:\/\/127\.0\.0\.1:\d+\/desktop$/); + const invocations = (await readFile(f.log, "utf8")) + .trim() + .split("\n") + .map((line) => JSON.parse(line) as string[]); + assert.equal(invocations.length, 2); + for (const args of invocations) { + assert.ok(args.includes("StrictHostKeyChecking=yes")); + assert.ok(args.includes("BatchMode=yes")); + assert.ok(!args.some((arg) => arg.includes(f.token))); + } + assert.ok(invocations[1]!.includes("ExitOnForwardFailure=yes")); + await tunnel.close(); + await assert.rejects( + readFile(output), + (error: NodeJS.ErrnoException) => error.code === "ENOENT", + ); + await tunnel.close(); + } finally { + await tunnel?.close(); + await f.close(); + } +}); +test("occupied local port receives no credential or readiness request", async () => { + const f = await fixture(); + let received = 0; + const occupied = httpServer((_req, res) => { + received++; + res.end("unrelated"); + }); + occupied.listen(0, "127.0.0.1"); + await once(occupied, "listening"); + try { + const output = join(f.directory, "must-not-exist.json"); + await assert.rejects( + openSSHTunnel({ + host: "mac", + remoteCredentialFile: "/private/viewer.json", + output, + port: (occupied.address() as { port: number }).port, + }), + /SSH closed before forwarding/, + ); + assert.equal(received, 0); + assert.equal(f.requests.length, 0); + await assert.rejects( + readFile(output), + (error: NodeJS.ErrnoException) => error.code === "ENOENT", + ); + } finally { + occupied.closeAllConnections(); + await new Promise((resolve) => occupied.close(() => resolve())); + await f.close(); + } +}); +test("authorization failure closes forwarding and leaves no credential", async () => { + const f = await fixture(true); + try { + const output = join(f.directory, "denied.json"); + const port = await freePort(); + await assert.rejects( + openSSHTunnel({ + host: "mac", + remoteCredentialFile: "/private/viewer.json", + output, + port, + }), + /Viewer scope only/, + ); + await assert.rejects( + readFile(output), + (error: NodeJS.ErrnoException) => error.code === "ENOENT", + ); + const probe = netServer(); + probe.listen(port, "127.0.0.1"); + await once(probe, "listening"); + await new Promise((resolve) => probe.close(() => resolve())); + } finally { + await f.close(); + } +}); +test("pre-existing output credentials are preserved on setup failure", async () => { + const f = await fixture(); + try { + const output = join(f.directory, "existing.json"); + await writeFile(output, "do not overwrite", { mode: 0o600 }); + await assert.rejects( + openSSHTunnel({ + host: "mac", + remoteCredentialFile: "/private/viewer.json", + output, + port: await freePort(), + }), + ); + assert.equal(await readFile(output, "utf8"), "do not overwrite"); + } finally { + await f.close(); + } +}); diff --git a/tests/desktop-view.test.ts b/tests/desktop-view.test.ts new file mode 100644 index 0000000..f93163e --- /dev/null +++ b/tests/desktop-view.test.ts @@ -0,0 +1,281 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; +import vm from "node:vm"; +import { desktopHTML } from "../src/desktop/view.js"; + +class Element { + textContent = ""; + value = ""; + hidden = false; + disabled = false; + className = ""; + style: Record = {}; + children: Element[] = []; + parent?: Element; + src?: string; + append(...nodes: Element[]) { + this.children.push(...nodes); + for (const node of nodes) node.parent = this; + } + replaceChildren(...nodes: Element[]) { + this.children = []; + this.append(...nodes); + } + get lastChild() { + const child = this.children.at(-1); + assert.ok(child); + return child; + } + remove() { + if (this.parent) + this.parent.children = this.parent.children.filter((n) => n !== this); + } + removeAttribute(key: string) { + if (key === "src") this.src = undefined; + } + focus() {} + blur() {} + setPointerCapture() {} + getBoundingClientRect() { + return { left: 0, top: 0, width: 100, height: 100 }; + } +} +function nextTimer(timers: Array<() => Promise | void>) { + const timer = timers.shift(); + assert.ok(timer); + return timer(); +} +const settle = () => new Promise((resolve) => setTimeout(resolve, 0)); +async function harness(admin = false) { + const elements = new Map(); + const element = (id: string) => { + if (!elements.has(id)) elements.set(id, new Element()); + const node = elements.get(id); + assert.ok(node); + return node; + }; + const calls: { method: string; args: Record }[] = []; + const timeouts: Array<() => Promise | void> = []; + const intervals: Array<() => Promise | void> = []; + let fail = false; + let controller: { participantId: string; subject?: string } | null = null; + const members: Array<{ + id: string; + name: string; + color: string; + cursor: { + x: number; + y: number; + target?: { kind: string; id?: number }; + } | null; + }> = []; + const document = { + onvisibilitychange: () => {}, + hidden: false, + getElementById: element, + createElement: () => new Element(), + }; + const listeners = new Map void>(); + const window = { + addEventListener: (name: string, callback: () => void) => + listeners.set(name, callback), + }; + const context = vm.createContext({ + AbortSignal, + document, + window, + location: { hash: "#token", pathname: "/view", search: "" }, + sessionStorage: { + getItem: () => null, + setItem: () => {}, + removeItem: () => {}, + }, + history: { replaceState: () => {} }, + crypto: { randomUUID: () => "request" }, + setTimeout: (f: () => Promise | void) => timeouts.push(f), + setInterval: (f: () => Promise | void) => intervals.push(f), + fetch: async (url: string, init: { body: string }) => { + if (fail) throw Error("Network lost"); + let data: unknown; + if (url.endsWith("/session")) + data = { + generation: "generation", + admin, + scopes: ["input-control", "viewer-read", "presence"], + }; + else if (url.endsWith("/frame")) + data = { + participants: members, + controller, + image: { width: 100, height: 100, base64: "png" }, + observationId: "observation", + }; + else { + const body = JSON.parse(init.body); + calls.push(body); + const response: { ok: boolean; result: unknown } = { + ok: true, + result: {}, + }; + data = response; + if (body.method === "presence.join") { + response.result = { + id: "self", + name: body.args.name, + color: "#fff", + cursor: null, + }; + members.push({ + id: "self", + name: body.args.name, + color: "#fff", + cursor: null, + }); + } + if (body.method === "takeover") { + response.result = { id: "lease" }; + controller = { participantId: "self" }; + } + if (body.method === "release") controller = null; + } + return { ok: true, json: async () => data }; + }, + }); + const match = desktopHTML("/api").match(/'); - }); - await new Promise(resolve => server.listen(0, '127.0.0.1', resolve)); - const address = server.address(); - assert.ok(address && typeof address !== 'string'); - const driver = new BrowserDriver({ executablePath: chrome, headless: true, args: ['--no-sandbox'] }); - try { - let s = await driver.snapshot(); - await driver.execute({ kind: 'navigate', text: `http://127.0.0.1:${address.port}` }, s.id); - s = await driver.snapshot(); - const input = s.nodes.find(n => n.name === 'Name'); assert.ok(input); - await driver.execute({ kind: 'setValue', ref: input.ref, text: 'Jev test' }, s.id); - await assert.rejects(driver.execute({ kind: 'setValue', ref: input.ref, text: 'duplicate' }, s.id), /Stale/); - s = await driver.snapshot(); - const save = s.nodes.find(n => n.name === 'Save'); assert.ok(save); - await driver.execute({ kind: 'press', ref: save.ref }, s.id); - s = await driver.snapshot(); - assert.ok(s.nodes.some(n => n.name === 'Saved: Jev test')); - const abort = new AbortController(); abort.abort(); - await assert.rejects(driver.execute({ kind: 'navigate', text: 'https://example.com' }, s.id, abort.signal)); - } finally { await driver.close(); await new Promise(resolve => server.close(() => resolve())); } -}); +const chrome = process.env.TEST_CHROME_PATH ?? "/usr/bin/google-chrome"; +(existsSync(chrome) ? test : test.skip)( + "DOM route fills a real form, observes its result, and rejects stale refs", + async () => { + const server = createServer((_req, res) => { + res.setHeader("Content-Type", "text/html"); + res.end( + 'Driver fixture

Waiting

', + ); + }); + await new Promise((resolve) => + server.listen(0, "127.0.0.1", resolve), + ); + const address = server.address(); + assert.ok(address && typeof address !== "string"); + const driver = new BrowserDriver({ + executablePath: chrome, + headless: true, + args: ["--no-sandbox"], + }); + try { + let s = await driver.snapshot(); + await driver.execute( + { kind: "navigate", text: `http://127.0.0.1:${address.port}` }, + s.id, + ); + s = await driver.snapshot(); + const input = s.nodes.find((n) => n.name === "Name"); + assert.ok(input); + await driver.execute( + { kind: "setValue", ref: input.ref, text: "Jev test" }, + s.id, + ); + await assert.rejects( + driver.execute( + { kind: "setValue", ref: input.ref, text: "duplicate" }, + s.id, + ), + /Stale/, + ); + s = await driver.snapshot(); + const save = s.nodes.find((n) => n.name === "Save"); + assert.ok(save); + await driver.execute({ kind: "press", ref: save.ref }, s.id); + s = await driver.snapshot(); + assert.ok(s.nodes.some((n) => n.name === "Saved: Jev test")); + const abort = new AbortController(); + abort.abort(); + await assert.rejects( + driver.execute( + { kind: "navigate", text: "https://example.com" }, + s.id, + abort.signal, + ), + ); + } finally { + await driver.close(); + await new Promise((resolve) => server.close(() => resolve())); + } + }, +); diff --git a/tests/desktop-http.test.ts b/tests/desktop-http.test.ts index 72a8726..88bd11b 100644 --- a/tests/desktop-http.test.ts +++ b/tests/desktop-http.test.ts @@ -8,14 +8,14 @@ import { desktopCall } from "../src/desktop/client.js"; import { startDesktop } from "../src/desktop/server.js"; // Exercise the real authenticated HTTP server without requiring an X server. -test("HTTP multiplayer isolates grants, publishes cursor state, and fences revoke/rebind", { - skip: process.platform !== "linux", -}, async () => { - const directory = await mkdtemp(join(tmpdir(), "opcode-http-")); - const helper = join(directory, "helper.cjs"); - await writeFile( - helper, - `#!/usr/bin/env node +(process.platform === "linux" ? test : test.skip)( + "HTTP multiplayer isolates grants, publishes cursor state, and fences revoke/rebind", + async () => { + const directory = await mkdtemp(join(tmpdir(), "opcode-http-")); + const helper = join(directory, "helper.cjs"); + await writeFile( + helper, + `#!/usr/bin/env node const op=process.argv[2]; if(op==='display') console.log(JSON.stringify({id:1,x:0,y:0,width:2,height:2})); else if(op==='state') console.log(JSON.stringify({focus:1,x:0,y:0})); @@ -23,120 +23,125 @@ else if(op==='capture') process.stdout.write(Buffer.concat([Buffer.from('P6\\n2 else if(op==='list') console.log('[]'); else console.log('{}'); `, - ); - await chmod(helper, 0o700); - let service: Awaited> | undefined; - try { - service = await startDesktop({ - display: `:${100000 + Math.floor(Math.random() * 100000000)}`, - uid: process.getuid!(), - generation: "one", - helper, - directory: join(directory, "data"), - }); - const endpoint = service.endpoint; - const host = JSON.parse(await readFile(service.path, "utf8")) - .token as string; - const rpc = async ( - token: string, - method: string, - args: Record = {}, - generation = "one", - ) => { - const response = await fetch(endpoint + "/rpc", { - method: "POST", - headers: { - Authorization: `Bearer ${token}`, - "Content-Type": "application/json", - }, - body: JSON.stringify({ id: randomUUID(), generation, method, args }), + ); + await chmod(helper, 0o700); + let service: Awaited> | undefined; + try { + service = await startDesktop({ + display: `:${100000 + Math.floor(Math.random() * 100000000)}`, + uid: process.getuid!(), + generation: "one", + helper, + directory: join(directory, "data"), }); - return { - status: response.status, - ...((await response.json()) as { - ok: boolean; - result: any; - error?: { code: string }; - }), + const endpoint = service.endpoint; + const host = JSON.parse(await readFile(service.path, "utf8")) + .token as string; + const rpc = async ( + token: string, + method: string, + args: Record = {}, + generation = "one", + ) => { + const response = await fetch(endpoint + "/rpc", { + method: "POST", + headers: { + Authorization: `Bearer ${token}`, + "Content-Type": "application/json", + }, + body: JSON.stringify({ id: randomUUID(), generation, method, args }), + }); + return { + status: response.status, + ...((await response.json()) as { + ok: boolean; + result: any; + error?: { code: string }; + }), + }; }; - }; - const alice = ( - await rpc(host, "grant", { - subject: "alice", - scopes: ["viewer-read", "presence"], - ttlMs: 60000, - }) - ).result; - const bob = ( - await rpc(host, "grant", { - subject: "bob", - scopes: ["viewer-read", "presence", "input-control"], - ttlMs: 60000, - }) - ).result; - const anonymous = await fetch(endpoint + "/frame"); - assert.equal(anonymous.status, 403); - const ap = (await rpc(alice.token, "presence.join", { name: "Alice" })) - .result; - const bp = (await rpc(bob.token, "presence.join", { name: "Bob" })).result; - assert.equal( - (await rpc(alice.token, "takeover", { participantId: ap.id })).status, - 403, - ); - assert.equal( - ( - await rpc(bob.token, "presence.update", { - participantId: ap.id, - cursor: { x: 0.2, y: 0.3 }, + const alice = ( + await rpc(host, "grant", { + subject: "alice", + scopes: ["viewer-read", "presence"], + ttlMs: 60000, }) - ).ok, - false, - ); - assert.equal( - ( - await rpc(alice.token, "presence.update", { - participantId: ap.id, - cursor: { x: 0.2, y: 0.3 }, + ).result; + const bob = ( + await rpc(host, "grant", { + subject: "bob", + scopes: ["viewer-read", "presence", "input-control"], + ttlMs: 60000, }) - ).ok, - true, - ); - const lease = (await rpc(bob.token, "takeover", { participantId: bp.id })) - .result; - assert.ok(lease.id); - const frameResponse = await fetch(endpoint + "/frame", { - headers: { Authorization: `Bearer ${alice.token}` }, - }); - assert.equal(frameResponse.status, 200); - const frame = (await frameResponse.json()) as any; - assert.equal(frame.controller.participantId, bp.id); - assert.deepEqual( - frame.participants.find((p: any) => p.id === ap.id).cursor, - { x: 0.2, y: 0.3 }, - ); - assert.ok(frame.image.base64); - assert.equal((await rpc(alice.token, "state")).result.lease.subject, "bob"); - await rpc(host, "revoke", { id: bob.id }); - const state = (await rpc(alice.token, "presence.list")).result; - assert.equal(state.controller, null); - assert.equal(state.participants.length, 1); - assert.equal((await rpc(bob.token, "state")).status, 403); - await rpc(host, "rebind", { generation: "two" }); - assert.equal((await rpc(alice.token, "state", {}, "two")).status, 403); - assert.deepEqual((await rpc(host, "presence.list", {}, "two")).result, { - participants: [], - controller: null, - }); - assert.deepEqual( - await desktopCall( - { endpoint, token: host, generation: "two" }, - "shutdown", - {}, - ), - { stopping: true }, - ); - } finally { - await service?.close(); - await rm(directory, { recursive: true, force: true }); - } -}); + ).result; + const anonymous = await fetch(endpoint + "/frame"); + assert.equal(anonymous.status, 403); + const ap = (await rpc(alice.token, "presence.join", { name: "Alice" })) + .result; + const bp = (await rpc(bob.token, "presence.join", { name: "Bob" })) + .result; + assert.equal( + (await rpc(alice.token, "takeover", { participantId: ap.id })).status, + 403, + ); + assert.equal( + ( + await rpc(bob.token, "presence.update", { + participantId: ap.id, + cursor: { x: 0.2, y: 0.3 }, + }) + ).ok, + false, + ); + assert.equal( + ( + await rpc(alice.token, "presence.update", { + participantId: ap.id, + cursor: { x: 0.2, y: 0.3 }, + }) + ).ok, + true, + ); + const lease = (await rpc(bob.token, "takeover", { participantId: bp.id })) + .result; + assert.ok(lease.id); + const frameResponse = await fetch(endpoint + "/frame", { + headers: { Authorization: `Bearer ${alice.token}` }, + }); + assert.equal(frameResponse.status, 200); + const frame = (await frameResponse.json()) as any; + assert.equal(frame.controller.participantId, bp.id); + assert.deepEqual( + frame.participants.find((p: any) => p.id === ap.id).cursor, + { x: 0.2, y: 0.3 }, + ); + assert.ok(frame.image.base64); + assert.equal( + (await rpc(alice.token, "state")).result.lease.subject, + "bob", + ); + await rpc(host, "revoke", { id: bob.id }); + const state = (await rpc(alice.token, "presence.list")).result; + assert.equal(state.controller, null); + assert.equal(state.participants.length, 1); + assert.equal((await rpc(bob.token, "state")).status, 403); + await rpc(host, "rebind", { generation: "two" }); + assert.equal((await rpc(alice.token, "state", {}, "two")).status, 403); + assert.deepEqual((await rpc(host, "presence.list", {}, "two")).result, { + participants: [], + controller: null, + }); + assert.deepEqual( + await desktopCall( + { endpoint, token: host, generation: "two" }, + "shutdown", + {}, + ), + { stopping: true }, + ); + } finally { + await service?.close(); + await rm(directory, { recursive: true, force: true }); + } + }, +); From a5e6606bf6453cc8aa4dd2858243c1aef75e57a7 Mon Sep 17 00:00:00 2001 From: Swaraj Bachu Date: Sat, 3 Oct 2026 22:24:55 +0000 Subject: [PATCH 04/10] fix: preserve held input and validate Mac keys before dispatch --- .github/workflows/shared-desktop.yml | 4 +- native/macos/Driver.swift | 5 +- scripts/test-macos-desktop.mjs | 160 +++++++++++++++++++++++++++ src/desktop/controller.ts | 8 +- src/desktop/mac-backend.ts | 32 ++++++ tests/desktop-controller.test.ts | 48 ++++++++ tests/desktop-http.test.ts | 11 ++ tests/mac-backend.test.ts | 25 +++++ 8 files changed, 288 insertions(+), 5 deletions(-) create mode 100644 scripts/test-macos-desktop.mjs diff --git a/.github/workflows/shared-desktop.yml b/.github/workflows/shared-desktop.yml index ba1b4a8..fefc46b 100644 --- a/.github/workflows/shared-desktop.yml +++ b/.github/workflows/shared-desktop.yml @@ -32,5 +32,7 @@ jobs: if: runner.os == 'macOS' env: CU_NATIVE_DIR: ${{ runner.temp }}/opcode-native - run: bash scripts/build-native.sh + run: | + bash scripts/build-native.sh + node scripts/test-macos-desktop.mjs - run: bun test diff --git a/native/macos/Driver.swift b/native/macos/Driver.swift index 29873d1..04a4463 100644 --- a/native/macos/Driver.swift +++ b/native/macos/Driver.swift @@ -23,6 +23,7 @@ struct SavedElement { var lastCaptureImage: CGImage? var virtualCursor: CGPoint? var desktopHeldModifiers: CGEventFlags = [] + let desktopKeyCodes: [String: CGKeyCode] = ["a":0,"s":1,"d":2,"f":3,"h":4,"g":5,"z":6,"x":7,"c":8,"v":9,"b":11,"q":12,"w":13,"e":14,"r":15,"y":16,"t":17,"1":18,"2":19,"3":20,"4":21,"6":22,"5":23,"9":25,"7":26,"8":28,"0":29,"o":31,"u":32,"i":34,"p":35,"l":37,"j":38,"k":40,"n":45,"m":46,"Enter":36,"Tab":48,"Space":49,"Backspace":51,"Escape":53,"Meta":55,"Shift":56,"Alt":58,"Control":59,"Home":115,"PageUp":116,"Delete":117,"End":119,"PageDown":121,"ArrowLeft":123,"ArrowRight":124,"ArrowDown":125,"ArrowUp":126,"F1":122,"F2":120,"F3":99,"F4":118,"F5":96,"F6":97,"F7":98,"F8":100,"F9":101,"F10":109,"F11":103,"F12":111] let foregroundAllowed = ProcessInfo.processInfo.environment["JEV_INTERACTION_MODE"] == "foreground" var activatedRenderers = Set() var observationErrors = 0 @@ -719,8 +720,7 @@ struct SavedElement { event.flags = desktopHeldModifiers event.setIntegerValueField(.mouseEventClickState, value: count); event.post(tap: .cghidEventTap) } - let codes: [String: CGKeyCode] = ["a":0,"s":1,"d":2,"f":3,"h":4,"g":5,"z":6,"x":7,"c":8,"v":9,"b":11,"q":12,"w":13,"e":14,"r":15,"y":16,"t":17,"1":18,"2":19,"3":20,"4":21,"6":22,"5":23,"9":25,"7":26,"8":28,"0":29,"o":31,"u":32,"i":34,"p":35,"l":37,"j":38,"k":40,"n":45,"m":46,"Enter":36,"Tab":48,"Space":49,"Backspace":51,"Escape":53,"Meta":55,"Shift":56,"Alt":58,"Control":59,"Home":115,"PageUp":116,"Delete":117,"End":119,"PageDown":121,"ArrowLeft":123,"ArrowRight":124,"ArrowDown":125,"ArrowUp":126,"F1":122,"F2":120,"F3":99,"F4":118,"F5":96,"F6":97,"F7":98,"F8":100,"F9":101,"F10":109,"F11":103,"F12":111] - func code(_ key: String) -> CGKeyCode? { codes[key] ?? (key.count == 1 ? codes[key.lowercased()] : nil) } + func code(_ key: String) -> CGKeyCode? { desktopKeyCodes[key] ?? (key.count == 1 ? desktopKeyCodes[key.lowercased()] : nil) } func keyboard(_ key: String, _ down: Bool, _ flags: CGEventFlags = []) throws { guard let c = code(key), let event = CGEvent(keyboardEventSource: nil, virtualKey: c, keyDown: down) else { throw DriverFailure(code: "unsupported", message: "Unsupported named key.") @@ -802,6 +802,7 @@ struct SavedElement { func handle(_ request: [String: Any]) async throws -> Any { switch request["method"] as? String { + case "desktopKeys": return desktopKeyCodes.keys.sorted() case "desktopGeometry": return try desktopGeometry(request) case "desktopWindows": return try desktopWindows() case "desktopState": return desktopState() diff --git a/scripts/test-macos-desktop.mjs b/scripts/test-macos-desktop.mjs new file mode 100644 index 0000000..08916da --- /dev/null +++ b/scripts/test-macos-desktop.mjs @@ -0,0 +1,160 @@ +import assert from "node:assert/strict"; +import { spawnSync } from "node:child_process"; +import { join, resolve } from "node:path"; + +if (process.platform !== "darwin") + throw new Error("This native smoke test requires macOS."); +if (!process.env.CU_NATIVE_DIR) + throw new Error( + "Set CU_NATIVE_DIR to the temporary compiled native output directory.", + ); +const binary = join(resolve(process.env.CU_NATIVE_DIR), "desktop-driver"); + +// Run only the temporary standalone binary. Never install, launch, stop, or +// reconnect the user's installed permission-owning helper. +function run(requests) { + const child = spawnSync(binary, [], { + input: requests.map((request) => JSON.stringify(request)).join("\n") + "\n", + encoding: "utf8", + timeout: 15_000, + maxBuffer: 1_000_000, + env: { ...process.env, JEV_INTERACTION_MODE: "background" }, + }); + if (child.error) throw child.error; + assert.equal( + child.signal, + null, + "Native driver unexpectedly terminated by a signal.", + ); + assert.equal( + child.status, + 0, + `Native driver exited unsuccessfully: ${child.stderr}`, + ); + const replies = child.stdout + .trim() + .split("\n") + .map((line) => JSON.parse(line)); + assert.equal( + replies.length, + requests.length, + "Expected exactly one JSONL response per request.", + ); + for (let index = 0; index < replies.length; index++) { + assert.equal( + replies[index].id, + requests[index].id, + "Native response ID mismatch.", + ); + assert.equal(typeof replies[index].ok, "boolean"); + } + return replies; +} +function denied(reply, codes) { + assert.equal(reply.ok, false, `${reply.id} must fail closed.`); + assert.ok( + codes.includes(reply.error?.code), + `${reply.id}: unexpected error code ${reply.error?.code}`, + ); + assert.equal(typeof reply.error.message, "string"); + assert.equal(reply.error.delivery, "notDispatched"); + assert.equal(reply.data, undefined); +} +const [statusReply, geometryReply, stateReply, keysReply] = run([ + { id: "status", method: "status" }, + { id: "geometry", method: "desktopGeometry", target: { kind: "display" } }, + { id: "state", method: "desktopState" }, + { id: "keys", method: "desktopKeys" }, +]); +assert.equal(statusReply.ok, true); +const status = statusReply.data; +assert.equal(status.platform, "macOS"); +assert.equal(typeof status.accessibility, "boolean"); +assert.equal(typeof status.screenRecording, "boolean"); +assert.equal(geometryReply.ok, true); +const geometry = geometryReply.data; +for (const key of ["id", "width", "height"]) { + assert.ok( + Number.isInteger(geometry[key]) && geometry[key] > 0, + `Invalid desktop geometry ${key}.`, + ); +} +for (const key of ["x", "y"]) + assert.ok(Number.isFinite(geometry[key]), `Invalid desktop geometry ${key}.`); +assert.equal(stateReply.ok, true); +assert.ok( + Number.isInteger(stateReply.data.focus) && stateReply.data.focus >= 0, +); +for (const key of ["x", "y"]) assert.ok(Number.isFinite(stateReply.data[key])); + +assert.equal(keysReply.ok, true); +assert.ok(Array.isArray(keysReply.data)); +assert.ok( + keysReply.data.every((key) => typeof key === "string" && key.length > 0), +); +assert.equal(new Set(keysReply.data).size, keysReply.data.length); +for (const key of ["a", "Enter", "Meta", "Control", "Shift", "Alt"]) + assert.ok(keysReply.data.includes(key)); + +const requests = [ + { id: "missing-target", method: "desktopGeometry" }, + { + id: "unknown-target", + method: "desktopGeometry", + target: { kind: "not-a-desktop" }, + }, + { id: "malformed-target", method: "desktopGeometry", target: "display" }, + { + id: "malformed-window", + method: "desktopGeometry", + target: { kind: "window", id: "not-an-id" }, + }, + { + id: "unknown-selector", + method: "desktopDoesNotExist", + target: { kind: "display" }, + }, + // With AX granted, explicit foreground approval still has to be present. + // This request can never produce valid input, even if permissions change. + { + id: "unapproved-input", + method: "desktopInput", + foregroundApproved: false, + target: { kind: "not-a-desktop" }, + action: { kind: "not-an-action" }, + }, +]; +// Capture with an invalid target cannot read private pixels even if permission +// becomes granted between status and this request. Never request valid capture. +if (!status.screenRecording) + requests.push({ + id: "capture-without-permission", + method: "desktopCapture", + target: { kind: "not-a-desktop" }, + }); +if (!status.accessibility) + requests.push({ + id: "input-without-permission", + method: "desktopInput", + foregroundApproved: false, + target: { kind: "not-a-desktop" }, + action: { kind: "not-an-action" }, + }); +const replies = run(requests); +for (const reply of replies) { + if (reply.id === "capture-without-permission") + denied(reply, ["permission_denied"]); + else if (reply.id === "input-without-permission") + denied(reply, ["AccessibilityDenied"]); + else if (reply.id === "unapproved-input") + denied(reply, ["AccessibilityDenied", "permission_denied"]); + else if (reply.id === "unknown-target" || reply.id === "malformed-window") + denied(reply, ["stale_observation"]); + else denied(reply, ["InvalidRequest"]); +} +console.log( + `PASS native Mac desktop selectors, geometry/state shapes, malformed targets, and permission/input approval gates (${4 + requests.length} requests; no screenshots or input dispatched).`, +); +console.log( + `Permission-dependent denial checks: Screen Recording ${status.screenRecording ? "skipped (already granted)" : "verified"}, Accessibility ${status.accessibility ? "skipped (already granted)" : "verified"}.`, +); diff --git a/src/desktop/controller.ts b/src/desktop/controller.ts index 0f6efb4..d82a7f3 100644 --- a/src/desktop/controller.ts +++ b/src/desktop/controller.ts @@ -374,6 +374,8 @@ export class DesktopController { () => abort.abort(), Math.max(1, Math.min(10000, currentLease.expiresAt - Date.now())), ); + const previouslyHeldKeys = new Set(this.heldKeys); + const previouslyHeldButtons = new Set(this.heldButtons); if (action.kind === "keyDown") this.heldKeys.add(action.key); if (action.kind === "buttonDown") this.heldButtons.add(action.button); const transientButtons = ["click", "doubleClick", "drag"].includes( @@ -396,8 +398,10 @@ export class DesktopController { await this.persistHeld(); await this.backend.input(record.value.target, action, abort.signal); for (const button of transientButtons) - this.heldButtons.delete(button); - for (const key of transientKeys) this.heldKeys.delete(key); + if (!previouslyHeldButtons.has(button)) + this.heldButtons.delete(button); + for (const key of transientKeys) + if (!previouslyHeldKeys.has(key)) this.heldKeys.delete(key); if (action.kind === "keyUp") this.heldKeys.delete(action.key); if (action.kind === "buttonUp") this.heldButtons.delete(action.button); diff --git a/src/desktop/mac-backend.ts b/src/desktop/mac-backend.ts index 7bedc58..586c862 100644 --- a/src/desktop/mac-backend.ts +++ b/src/desktop/mac-backend.ts @@ -30,6 +30,7 @@ export class MacBackend implements DesktopBackend { "Local applications and physical input bypass broker arbitration.", ], }; + private supportedKeys?: Promise>; constructor(readonly driver: MacDriver) {} async geometry(target: Target): Promise { return GeometrySchema.parse( @@ -81,6 +82,37 @@ export class MacBackend implements DesktopBackend { return { geometry: result.geometry, png, ppm: Buffer.alloc(0), sample }; } async prepareAction(action: Action) { + if ( + action.kind === "key" || + action.kind === "keyDown" || + action.kind === "keyUp" + ) { + // Query the native source of truth before the controller journals held input. + this.supportedKeys ??= this.driver + .request("desktopKeys") + .then( + (value) => + new Set( + z.array(z.string().min(1).max(100)).min(1).max(256).parse(value), + ), + ); + const supported = await this.supportedKeys; + const parts = action.key.split("+"); + const modifiers = new Set(["Meta", "Control", "Alt", "Shift"]); + if ( + parts.some( + (key) => + !supported.has(key) && + !(key.length === 1 && supported.has(key.toLowerCase())), + ) || + (action.kind !== "key" && parts.length !== 1) || + parts.slice(0, -1).some((key) => !modifiers.has(key)) + ) + throw failure( + "unsupported", + "Unsupported Mac key or modifier combination.", + ); + } return action; } async input(target: Target, action: Action, signal: AbortSignal) { diff --git a/tests/desktop-controller.test.ts b/tests/desktop-controller.test.ts index 4a507d5..6031fc3 100644 --- a/tests/desktop-controller.test.ts +++ b/tests/desktop-controller.test.ts @@ -341,3 +341,51 @@ test("failed key release remains journaled and blocks acquisition until restart await rm(directory, { recursive: true, force: true }); } }); + +test("transient shortcuts and clicks preserve preexisting held input until explicit release or takeover", async () => { + const directory = await mkdtemp(join(tmpdir(), "opcode-held-input-")); + const backend = new Fixture("", {}); + const c = new DesktopController(backend, "one", directory); + await c.init(); + try { + const grant = c.authority.issue("agent", [...scopes], 60000); + const human = c.authority.issue("human", [...scopes], 60000); + const lease = (await c.call(request("acquire"), grant)) as { id: string }; + const input = async (action: Action) => { + const observation = await c.observe(grant, { kind: "display" }); + await c.call( + request("input", { + leaseId: lease.id, + observationId: observation.observationId, + action, + }), + grant, + ); + }; + const held = async () => + JSON.parse(await readFile(join(directory, "held-input.json"), "utf8")); + await input({ kind: "keyDown", key: "Control" }); + await input({ kind: "buttonDown", button: 1 }); + await input({ kind: "key", key: "Control+a" }); + await input({ kind: "text", text: "hello", pasteKey: "Control+v" }); + await input({ kind: "click", x: 1, y: 1 }); + assert.deepEqual(await held(), { keys: ["Control"], buttons: [1] }); + await input({ kind: "keyUp", key: "Control" }); + await input({ kind: "buttonUp", button: 1 }); + assert.deepEqual(await held(), { keys: [], buttons: [] }); + await input({ kind: "keyDown", key: "Control" }); + await input({ kind: "buttonDown", button: 1 }); + await input({ kind: "key", key: "Control+a" }); + await input({ kind: "click", x: 1, y: 1 }); + await c.call(request("takeover"), human); + assert.equal(backend.releaseCount, 2); + assert.equal( + backend.delivered.filter((action) => action.kind === "buttonUp").length, + 2, + ); + assert.deepEqual(await held(), { keys: [], buttons: [] }); + } finally { + await c.close(); + await rm(directory, { recursive: true, force: true }); + } +}); diff --git a/tests/desktop-http.test.ts b/tests/desktop-http.test.ts index 88bd11b..df8463d 100644 --- a/tests/desktop-http.test.ts +++ b/tests/desktop-http.test.ts @@ -31,6 +31,7 @@ else console.log('{}'); display: `:${100000 + Math.floor(Math.random() * 100000000)}`, uid: process.getuid!(), generation: "one", + origin: "http://127.0.0.1:4311", helper, directory: join(directory, "data"), }); @@ -74,6 +75,16 @@ else console.log('{}'); ttlMs: 60000, }) ).result; + for (const [origin, expected] of [ + ["http://127.0.0.1:4311", 200], + [new URL(endpoint).origin, 200], + ["https://untrusted.example", 403], + ] as const) { + const response = await fetch(endpoint + "/session", { + headers: { Authorization: `Bearer ${alice.token}`, Origin: origin }, + }); + assert.equal(response.status, expected); + } const anonymous = await fetch(endpoint + "/frame"); assert.equal(anonymous.status, 403); const ap = (await rpc(alice.token, "presence.join", { name: "Alice" })) diff --git a/tests/mac-backend.test.ts b/tests/mac-backend.test.ts index e0929e1..3915005 100644 --- a/tests/mac-backend.test.ts +++ b/tests/mac-backend.test.ts @@ -101,3 +101,28 @@ test("Mac launch accepts only installed bundle IDs and resize is explicit", asyn await assert.rejects(backend.resize(1000, 800), /not supported/); assert.equal(backend.capabilities.resize, false); }); +test("Mac unsupported held keys fail during preparation before native dispatch", async () => { + const { backend, calls, replies } = fixture(); + replies.desktopKeys = ["a", "Meta", "Control", "Alt", "Shift", "Enter"]; + await assert.rejects( + backend.prepareAction({ kind: "keyDown", key: "NotAKey" }), + /Unsupported Mac key/, + ); + await assert.rejects( + backend.prepareAction({ kind: "key", key: "a+Enter" }), + /Unsupported Mac key/, + ); + await assert.rejects( + backend.prepareAction({ kind: "keyDown", key: "Meta+a" }), + /Unsupported Mac key/, + ); + assert.deepEqual( + await backend.prepareAction({ kind: "key", key: "Meta+A" }), + { kind: "key", key: "Meta+A" }, + ); + assert.deepEqual( + await backend.prepareAction({ kind: "keyUp", key: "Meta" }), + { kind: "keyUp", key: "Meta" }, + ); + assert.deepEqual(calls, [{ method: "desktopKeys", args: {} }]); +}); From ba00c8da0402d5039f6165e3cac1e64a8caeb345 Mon Sep 17 00:00:00 2001 From: Swaraj Bachu Date: Sat, 3 Oct 2026 22:24:55 +0000 Subject: [PATCH 05/10] docs: align remote setup and platform behavior with implementation --- README.md | 8 +++++--- docs/DESKTOP_SERVICE.md | 32 ++++++++++++++++++++------------ docs/HARNESS_SETUP.md | 7 ++++++- docs/MAC_DESKTOP.md | 10 +++++++--- docs/ZUSE_INTEGRATION.md | 13 ++++++++----- 5 files changed, 46 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index cba0f2a..2416d50 100644 --- a/README.md +++ b/README.md @@ -118,9 +118,11 @@ A new session is another way to refresh discovery. The skill teaches the agent t find or launch the right app, choose exact controls, verify results, and recover from local blockers while retaining ownership of the full workflow. You should not have to dictate each click. -**A cloud coding workspace needs a Mac command bridge supplied by its host.** -Install and execute Jev on the Mac through that bridge. A Linux shell or cloud MCP -process alone cannot operate your Mac. For shared remote desktops, use the scoped `cu desktop-api` SSH/HTTPS bridge described below. +**A cloud coding workspace can use the scoped `cu desktop-api` SSH/HTTPS bridge** +to operate an existing Mac desktop; see [remote desktops and multiplayer](#remote-desktops-and-multiplayer). +The Mac must run the broker and permission-owning helper. The semantic `cu observe` +and `cu execute` commands above still run on the Mac, through shell access or a +command bridge supplied by the host. ## Optional delegated workflows diff --git a/docs/DESKTOP_SERVICE.md b/docs/DESKTOP_SERVICE.md index 19d96a6..1f9f20b 100644 --- a/docs/DESKTOP_SERVICE.md +++ b/docs/DESKTOP_SERVICE.md @@ -6,7 +6,7 @@ No reasoning model or model key is needed. Linux X11 and macOS are implemented b coordinates and limitations, see [Shared Mac desktop](MAC_DESKTOP.md). This is not a Wayland, Windows, audio or multi-monitor desktop implementation. -## Install and attach +## Linux install and attach ```sh npm install --omit=optional @opcodehq/cu @@ -85,15 +85,21 @@ Actions: `click`, `doubleClick`, `rightClick`, `hover` with `x,y`; `drag` adds `toX,toY`; `scroll` has `amount` −30…30 and `axis` x/y; `text` has UTF-8 `text` (up to 8000 characters), with optional `pasteKey: "Control+Shift+v"` or `"Shift+Insert"` for terminals (known terminal classes select the matching shortcut automatically; other apps use Control+v); `key` uses names/combinations such as `Control+a`; `keyDown`/`keyUp` hold one key; `buttonDown`/`buttonUp` take button 1–3; -`focus` takes `windowId`; `launch` takes an installed `.desktop` `appId`. -Launching uses the desktop entry through `gio`, not caller-supplied shell text. +`focus` takes `windowId`. On Linux, `launch` takes an installed `.desktop` +`appId` and uses its desktop entry through `gio`, not caller-supplied shell text. +On macOS, `appId` is an installed application bundle ID. `pasteKey` applies only +to Linux; Mac text uses Unicode events. Coordinates are pixels in the delivered image, with explicit identity scale and root offsets in `imageToDesktop`. No implicit crops/downscaling/CSS coordinates. -Full-display input respects keyboard focus. Window-target input raises/focuses -that window, matching the legacy isolated X11 behavior. Text pastes through that -X display's clipboard and replaces its previous selection (Shift+Insert also sets PRIMARY); preservation is not -promised. Verify the resulting app content. +Full-display input respects keyboard focus. On Linux, window-target input +raises/focuses that window, matching the legacy isolated X11 behavior. Text +pastes through that X display's clipboard and replaces its previous selection +(Shift+Insert also sets PRIMARY); preservation is not promised. On macOS, focus +the target window and obtain a fresh observation before physical input. Mac +captures use logical screen points, including on Retina displays; see +[Mac coordinates and limits](MAC_DESKTOP.md#control-and-coordinates). Verify the +resulting app content. Observations are grant-bound, expire after 30 seconds and are consumed by input. Geometry, focus and a sampled pixel-difference check reject changed scenes. @@ -104,18 +110,20 @@ and 512 observations overall. Observations older than 30 seconds are evicted. Du 512-entry cache; reuse with different content is rejected. Never retry uncertain input automatically or promise exactly-once delivery across crashes. -One broker serves all new MCP, CLI and viewer clients. While it is registered, -legacy window input and browser mutations on that display fail closed instead +One broker serves all `desktop-api` MCP, CLI and viewer clients. On Linux, while +it is registered, legacy window input and browser mutations on that display fail closed instead of opening a second control path. Existing window/browser workflows still work on displays without a broker. The native helper and arbitrary X clients remain -outside this application-level trust boundary. +outside this application-level trust boundary. Mac semantic sessions and local +hardware input also remain outside the shared broker; see the +[Mac control boundaries](MAC_DESKTOP.md#boundaries-and-recovery). ## Viewing and reverse proxies GET `/view#TOKEN` opens the standalone viewer. Fragment tokens are removed from the address bar and held in sessionStorage; API requests use -Authorization headers. Add `?windowId=ID` before the fragment for window-only viewing. A viewer grant needs `viewer-read`; human control also -needs `input-control`. Observer buttons are hidden and mutations are rejected +Authorization headers. Add `?windowId=ID` before the fragment for window-only viewing. A viewer grant needs `viewer-read`; joining and showing a cursor also need +`presence`, and human control needs `input-control`. Observer buttons are hidden and mutations are rejected server-side. Never use the host master token as a viewer link. For embedding, use `/session` for generation/scopes, `/frame` for a PNG observation, diff --git a/docs/HARNESS_SETUP.md b/docs/HARNESS_SETUP.md index 2f145df..31064af 100644 --- a/docs/HARNESS_SETUP.md +++ b/docs/HARNESS_SETUP.md @@ -8,6 +8,11 @@ Exact native tools work without a TypeSafe key; Jev is optional. `cu task --provider NAME --model ID` runs full workflows through Vercel AI SDK; `cu providers` lists the built-in provider routes and credential variables. +For shared Mac/Linux desktops, remote SSH/HTTPS access, and multiplayer, use +the npm CLI’s `cu desktop-api` commands. The standalone bundle described below +does not include those commands. See [shared desktop setup](DESKTOP_SERVICE.md) +and [Mac setup](MAC_DESKTOP.md). + ## First installation Install an extracted standalone Mac bundle with its `install.sh`, then: @@ -89,7 +94,7 @@ subscription-backed agent to use native computer tools. | Symptom | Fix | | --- | --- | | Setup exits 2 | Installation succeeded but required readiness checks remain. Follow the printed fixes and rerun `jev doctor`. | -| No Mac / Linux platform | Run through the harness's existing Mac command bridge or use a local Mac harness. Installing locally in the cloud is insufficient. | +| Mac semantic tools from Linux/cloud | Run semantic commands through a Mac shell/command bridge, or use the separate `cu desktop-api` SSH/HTTPS workflow for shared desktop capture and input. The target Mac must run its helper and broker. | | Accessibility missing | Request it from the same command host that will run Jev; grant the host macOS identifies and restart it if requested. | | Key missing after saving | Check the command host's user and `JEV_SETTINGS_PATH`; the app and shell must read the same settings file. | | Capture works in Electron but fails from CLI | Screen Recording grants may differ by host. Grant the actual terminal/agent host for visual tasks. | diff --git a/docs/MAC_DESKTOP.md b/docs/MAC_DESKTOP.md index 5bf6c1f..e4669d7 100644 --- a/docs/MAC_DESKTOP.md +++ b/docs/MAC_DESKTOP.md @@ -7,10 +7,13 @@ created. Apps already signed in on the target Mac keep their existing sessions. ## Prepare the target Mac -Use macOS 14 or newer with an active graphical login. Install this version of CU -and its permission-owning helper: +Use macOS 14 or newer with an active graphical login, Node.js 20+, Bun 1.3.10+, +and Xcode Command Line Tools. Install the npm CLI build containing these commands and its +permission-owning helper; the legacy standalone bundle does not contain +`desktop-api`: ```sh +# Install the package built from this version, then initialize the Mac helper: cu install cu doctor ``` @@ -60,7 +63,8 @@ cu desktop-api call --display macos --method revoke --args '{"id":"GRANT_ID"}' For another computer, establish normal SSH authentication and verify the target's host key first. A Tailnet SSH hostname or an SSH configuration alias works. On the -client computer, keep this tunnel command running: +client computer, install the same npm CLI build (Node.js 20+); remote clients do +not need to install a local native helper. Keep this tunnel command running: ```sh cu desktop-api tunnel --ssh user@my-mac \ diff --git a/docs/ZUSE_INTEGRATION.md b/docs/ZUSE_INTEGRATION.md index c4a5c4c..6fe5a37 100644 --- a/docs/ZUSE_INTEGRATION.md +++ b/docs/ZUSE_INTEGRATION.md @@ -22,10 +22,12 @@ and recordings. The agent keeps its own model and subscription. 5. On restore or fork, call `rebind` with a new generation before allowing access. On workspace shutdown, call `shutdown`; the provider owns the desktop itself. -For an agent, grant `observe`, `input-control`, and optionally `viewer-read` for -state inspection. Recording scopes are separate. For a person watching, grant -`viewer-read`; add `input-control` only when they may take over. Keep host/admin -credentials in Zuse's backend. Grant tokens expire and can be revoked early. +For an agent, grant `observe`, `input-control`, `viewer-read`, and `presence` +for the multiplayer MCP workflow. Recording scopes are separate. For a person +watching with a named cursor, grant `viewer-read` and `presence`; add +`input-control` only when they may take over. The `share --role agent`, +`share --role viewer`, and `share --role controller` commands create these scope +sets. Keep host/admin credentials in Zuse's backend. Grant tokens expire and can be revoked early. The agent loop is **acquire → observe → input → observe**. Renew its lease while working. A person taking control interrupts agent input; after control is returned, @@ -35,7 +37,8 @@ click or text action is never automatically repeated. The complete commands, credential format, HTTP methods, errors and limitations are in [the service contract](DESKTOP_SERVICE.md). A host proxy example is in [desktop-proxy.mjs](../examples/desktop-proxy.mjs). Use a stable scoped grant per -viewer session; Zuse still validates its own login on every proxy request. +viewer session; configure the broker with the exact external viewer origin +and any proxy base path. Zuse still validates its own login on every proxy request. ## What still needs a Zuse test From 02765d5d487f031418880fcfb10e3a10bd962bd2 Mon Sep 17 00:00:00 2001 From: Swaraj Bachu Date: Sat, 3 Oct 2026 22:34:54 +0000 Subject: [PATCH 06/10] test: verify live Mac multiplayer input against disposable CI app --- .github/workflows/shared-desktop.yml | 1 + docs/MAC_DESKTOP.md | 4 +- native/macos/Driver.swift | 6 + scripts/test-macos-desktop-live.ts | 328 +++++++++++++++++++++++++++ tests/fixtures/mac-desktop.swift | 60 +++++ 5 files changed, 398 insertions(+), 1 deletion(-) create mode 100644 scripts/test-macos-desktop-live.ts create mode 100644 tests/fixtures/mac-desktop.swift diff --git a/.github/workflows/shared-desktop.yml b/.github/workflows/shared-desktop.yml index fefc46b..e3c4415 100644 --- a/.github/workflows/shared-desktop.yml +++ b/.github/workflows/shared-desktop.yml @@ -35,4 +35,5 @@ jobs: run: | bash scripts/build-native.sh node scripts/test-macos-desktop.mjs + bun scripts/test-macos-desktop-live.ts - run: bun test diff --git a/docs/MAC_DESKTOP.md b/docs/MAC_DESKTOP.md index e4669d7..4a2ecda 100644 --- a/docs/MAC_DESKTOP.md +++ b/docs/MAC_DESKTOP.md @@ -112,7 +112,9 @@ observation includes the target's desktop origin, which can be negative for a window on another display. Full-display capture selects the primary display. Use the window list and its IDs to select an exact window. Before physical input to a window target, focus that window and obtain a fresh observation; otherwise -input is rejected. Window capture does not provide an isolated input session. +input is rejected. Before scrolling or pressing a held mouse button on a window, +hover inside that window and observe again; a pointer outside the target is rejected. +Window capture does not provide an isolated input session. Supported input includes clicks, double/right clicks, hover, drag, vertical and horizontal scroll, text, named keys and modifier combinations, held keys/buttons, diff --git a/native/macos/Driver.swift b/native/macos/Driver.swift index 04a4463..391f1d5 100644 --- a/native/macos/Driver.swift +++ b/native/macos/Driver.swift @@ -706,6 +706,12 @@ struct SavedElement { } let origin = CGPoint(x: geometry["x"] as? Double ?? 0, y: geometry["y"] as? Double ?? 0) let width = geometry["width"] as! Int, height = geometry["height"] as! Int + if (request["target"] as? [String: Any])?["kind"] as? String == "window", ["scroll", "buttonDown"].contains(kind) { + let bounds = CGRect(x: origin.x, y: origin.y, width: Double(width), height: Double(height)) + guard let cursor = CGEvent(source: nil)?.location, bounds.contains(cursor) else { + throw DriverFailure(code: "stale_observation", message: "Hover inside the target window and observe again before scrolling or holding a button.") + } + } func point(_ x: String, _ y: String) throws -> CGPoint { guard let px = action[x] as? Double, let py = action[y] as? Double, px.isFinite, py.isFinite, px >= 0, py >= 0, px < Double(width), py < Double(height) else { diff --git a/scripts/test-macos-desktop-live.ts b/scripts/test-macos-desktop-live.ts new file mode 100644 index 0000000..38732d0 --- /dev/null +++ b/scripts/test-macos-desktop-live.ts @@ -0,0 +1,328 @@ +import assert from "node:assert/strict"; +import { + type ChildProcessWithoutNullStreams, + spawn, + spawnSync, +} from "node:child_process"; +import { randomUUID } from "node:crypto"; +import { mkdtemp, readFile, realpath, rm } from "node:fs/promises"; +import { join, relative, resolve } from "node:path"; +import { createInterface } from "node:readline"; +import { setTimeout as delay } from "node:timers/promises"; +import type { Grant } from "../src/desktop/authority.js"; +import { DesktopController } from "../src/desktop/controller.js"; +import { MacBackend, type MacDriver } from "../src/desktop/mac-backend.js"; +import { + type Action, + type Observation, + scopes, +} from "../src/desktop/protocol.js"; + +if ( + process.platform !== "darwin" || + process.env.CI !== "true" || + process.env.GITHUB_ACTIONS !== "true" +) + throw Error( + "Live Mac desktop tests run only on disposable GitHub macOS CI runners.", + ); +if (!process.env.RUNNER_TEMP || !process.env.CU_NATIVE_DIR) + throw Error( + "Set RUNNER_TEMP and CU_NATIVE_DIR to the CI temporary build directory.", + ); +const runner = await realpath(process.env.RUNNER_TEMP); +const native = await realpath(process.env.CU_NATIVE_DIR); +const withinRunner = relative(runner, native); +if ( + !withinRunner || + withinRunner.startsWith("..") || + withinRunner.startsWith("/") +) + throw Error( + "CU_NATIVE_DIR must be a temporary build directory beneath RUNNER_TEMP.", + ); +const directory = await mkdtemp(join(runner, "opcode-live-")); +const driverProcess = spawn(join(native, "desktop-driver"), [], { + stdio: ["pipe", "pipe", "pipe"], + env: { ...process.env, JEV_INTERACTION_MODE: "background" }, +}); +let fixtureProcess: ChildProcessWithoutNullStreams | undefined; +let controller: DesktopController | undefined; +let heartbeat: ReturnType | undefined; +let diagnostics = ""; +driverProcess.stderr.on("data", (chunk) => { + diagnostics = (diagnostics + chunk).slice(-16_000); +}); +const pending = new Map< + string, + { + resolve(value: unknown): void; + reject(error: Error): void; + timer: ReturnType; + } +>(); +function disconnected() { + for (const task of pending.values()) { + clearTimeout(task.timer); + task.reject(Error("Temporary native driver disconnected. " + diagnostics)); + } + pending.clear(); +} +driverProcess.on("error", disconnected); +driverProcess.on("exit", disconnected); +const lines = createInterface({ input: driverProcess.stdout }); +lines.on("line", (line) => { + try { + const reply = JSON.parse(line); + const task = pending.get(reply.id); + if (!task) throw Error("Unexpected native reply ID."); + pending.delete(reply.id); + clearTimeout(task.timer); + if (reply.ok) task.resolve(reply.data); + else task.reject(Object.assign(Error(reply.error.message), reply.error)); + } catch (error) { + disconnected(); + driverProcess.kill(); + } +}); +const driver: MacDriver = { + async request(method, args = {}, signal) { + signal?.throwIfAborted(); + const id = randomUUID(); + const result = await new Promise((resolve, reject) => { + const timer = setTimeout(() => { + pending.delete(id); + reject(Error(`Native ${method} timed out.`)); + driverProcess.kill(); + }, 15_000); + pending.set(id, { resolve, reject, timer }); + driverProcess.stdin.write(JSON.stringify({ ...args, id, method }) + "\n"); + }); + signal?.throwIfAborted(); + return result; + }, +}; +async function poll( + read: () => Promise, + message: string, +): Promise { + for (let attempt = 0; attempt < 50; attempt++) { + const value = await read(); + if (value !== undefined) return value; + await delay(100); + } + throw Error(message); +} +async function stop(child: ChildProcessWithoutNullStreams | undefined) { + if (!child || child.exitCode !== null || child.signalCode !== null) return; + await new Promise((resolve) => { + const force = setTimeout(() => child.kill("SIGKILL"), 1000); + child.once("exit", () => { + clearTimeout(force); + resolve(); + }); + child.kill("SIGTERM"); + }); +} +try { + const status = (await driver.request("status")) as { + accessibility: boolean; + screenRecording: boolean; + }; + assert.equal( + status.accessibility, + true, + "Live CI requires Accessibility; native delivery is unverified without it.", + ); + assert.equal( + status.screenRecording, + true, + "Live CI requires Screen Recording; native capture is unverified without it.", + ); + const fixtureBinary = join(directory, "fixture"); + const compile = spawnSync( + "xcrun", + [ + "swiftc", + "-swift-version", + "5", + "-parse-as-library", + "-framework", + "AppKit", + resolve("tests/fixtures/mac-desktop.swift"), + "-o", + fixtureBinary, + ], + { encoding: "utf8", timeout: 120_000 }, + ); + assert.equal(compile.status, 0, compile.stderr || compile.error?.message); + const title = "Opcode CI " + randomUUID(); + const output = join(directory, "result.json"); + fixtureProcess = spawn(fixtureBinary, [output, title], { + stdio: ["pipe", "pipe", "pipe"], + }); + fixtureProcess.on("error", () => {}); + fixtureProcess.stderr.on("data", (chunk) => { + diagnostics = (diagnostics + chunk).slice(-16_000); + }); + const manifest = await poll(async () => { + try { + return JSON.parse(await readFile(output + ".ready", "utf8")) as { + pid: number; + field: { x: number; y: number }; + save: { x: number; y: number }; + }; + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + return undefined; + } + }, "Fixture did not become ready. " + diagnostics); + const backend = new MacBackend(driver); + const window = await poll( + async () => + (await backend.windows()).find( + (window) => window.title === title && window.pid === manifest.pid, + ), + "Exact fixture window was not found.", + ); + controller = new DesktopController( + backend, + "live-ci", + join(directory, "recordings"), + ); + await controller.init(); + const c = controller; + const call = ( + grant: Grant, + method: string, + args: Record = {}, + ) => c.call({ id: randomUUID(), generation: "live-ci", method, args }, grant); + const agent = c.authority.issue("agent", [...scopes], 60_000); + const human = c.authority.issue("human", [...scopes], 60_000); + const a = (await call(agent, "presence.join", { + name: "CI agent", + role: "agent", + })) as { id: string }; + const h = (await call(human, "presence.join", { + name: "CI human", + role: "human", + })) as { id: string }; + heartbeat = setInterval(() => { + void call(agent, "presence.update", { participantId: a.id }).catch( + () => {}, + ); + void call(human, "presence.update", { participantId: h.id }).catch( + () => {}, + ); + }, 3000); + const before = await backend.state(); + await call(agent, "presence.update", { + participantId: a.id, + cursor: { x: 0.2, y: 0.3 }, + }); + await call(human, "presence.update", { + participantId: h.id, + cursor: { x: 0.7, y: 0.8 }, + }); + const after = await backend.state(); + assert.deepEqual( + { x: after.x, y: after.y }, + { x: before.x, y: before.y }, + "Overlay cursors must not move the real Mac pointer.", + ); + const presence = (await call(human, "presence.list")) as { + participants: { cursor: unknown }[]; + }; + assert.equal(presence.participants.length, 2); + assert.deepEqual( + presence.participants.map((participant) => participant.cursor), + [ + { x: 0.2, y: 0.3 }, + { x: 0.7, y: 0.8 }, + ], + ); + const firstLease = (await call(agent, "acquire", { + participantId: a.id, + })) as { id: string }; + await assert.rejects( + call(human, "acquire", { participantId: h.id }), + /controller/, + ); + const old = await c.observe(agent, { kind: "display" }); + const lease = (await call(human, "takeover", { participantId: h.id })) as { + id: string; + }; + await assert.rejects( + call(agent, "input", { + leaseId: firstLease.id, + observationId: old.observationId, + action: { kind: "hover", x: 1, y: 1 }, + }), + /lease|control/i, + ); + // Focus only our exact disposable window before delivering physical input. + const observed = await c.observe(human, { kind: "display" }); + await call(human, "input", { + leaseId: lease.id, + observationId: observed.observationId, + action: { kind: "focus", windowId: window.id }, + }); + await poll( + async () => + (await backend.state()).focus === window.id ? true : undefined, + "Fixture window did not become focused.", + ); + const target = { kind: "window" as const, id: window.id }; + async function act(action: Action | ((observation: Observation) => Action)) { + await delay(200); + const observation = await c.observe(human, target); + const png = Buffer.from(observation.image.base64, "base64"); + assert.equal(png.readUInt32BE(16), observation.image.width); + assert.equal(png.readUInt32BE(20), observation.image.height); + assert.equal(observation.imageToDesktop.scaleX, 1); + assert.equal(observation.imageToDesktop.scaleY, 1); + await call(human, "input", { + leaseId: lease.id, + observationId: observation.observationId, + action: typeof action === "function" ? action(observation) : action, + }); + } + const click = + (point: { x: number; y: number }) => + (observation: Observation): Action => ({ + kind: "click", + x: point.x - observation.desktopBounds.x, + y: point.y - observation.desktopBounds.y, + }); + await act(click(manifest.field)); + const text = "Opcode shared desktop CI Unicode α🙂 and chunked text verified"; + await act({ kind: "text", text }); + await act(click(manifest.save)); + const result = await poll(async () => { + try { + return JSON.parse(await readFile(output, "utf8")); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + return undefined; + } + }, "Native save click did not produce fixture output."); + assert.deepEqual( + result, + { saved: true, text }, + "Native text/click outcome must match exactly.", + ); + await call(human, "release", { leaseId: lease.id }); + console.log( + "PASS real Mac capture/point geometry, two independent overlay cursors, exclusive control/takeover, native focus/click/Unicode text/save against disposable fixture.", + ); +} finally { + if (heartbeat) clearInterval(heartbeat); + try { + await controller?.close(); + } finally { + lines.close(); + await stop(driverProcess); + await stop(fixtureProcess); + await rm(directory, { recursive: true, force: true }); + } +} diff --git a/tests/fixtures/mac-desktop.swift b/tests/fixtures/mac-desktop.swift new file mode 100644 index 0000000..4df4b06 --- /dev/null +++ b/tests/fixtures/mac-desktop.swift @@ -0,0 +1,60 @@ +import AppKit +import Foundation + +// Disposable CI-only target. No user files, network access, or installed helper. +@MainActor final class Fixture: NSObject, NSApplicationDelegate { + let output: String + let title: String + var window: NSWindow! + var field: NSTextField! + var save: NSButton! + init(output: String, title: String) { self.output = output; self.title = title } + func applicationDidFinishLaunching(_ notification: Notification) { + window = NSWindow(contentRect: NSRect(x: 100, y: 100, width: 480, height: 240), styleMask: [.titled, .closable], backing: .buffered, defer: false) + window.title = title + window.isReleasedWhenClosed = false + field = NSTextField(frame: NSRect(x: 40, y: 150, width: 390, height: 28)) + field.placeholderString = "CI text fixture" + save = NSButton(frame: NSRect(x: 40, y: 80, width: 140, height: 32)) + save.title = "Save fixture" + save.bezelStyle = .rounded + save.target = self; save.action = #selector(saveFixture) + window.contentView!.addSubview(field) + window.contentView!.addSubview(save) + window.makeKeyAndOrderFront(nil) + NSApp.activate(ignoringOtherApps: true) + window.makeFirstResponder(field) + DispatchQueue.main.asyncAfter(deadline: .now() + 0.5) { self.ready() } + } + func center(_ view: NSView) -> [String: Double] { + let rect = window.convertToScreen(view.convert(view.bounds, to: nil)) + let primaryTop = NSScreen.screens[0].frame.maxY + return ["x": rect.midX, "y": primaryTop - rect.midY] + } + func ready() { + let manifest: [String: Any] = ["title": title, "pid": Int(ProcessInfo.processInfo.processIdentifier), "field": center(field), "save": center(save)] + do { + let data = try JSONSerialization.data(withJSONObject: manifest) + try data.write(to: URL(fileURLWithPath: output + ".ready"), options: .atomic) + } catch { fputs("Could not write fixture readiness.\n", stderr); exit(1) } + } + @objc func saveFixture() { + do { + let data = try JSONSerialization.data(withJSONObject: ["text": field.stringValue, "saved": true]) + try data.write(to: URL(fileURLWithPath: output), options: .atomic) + } catch { fputs("Could not write fixture outcome.\n", stderr); exit(1) } + } +} + +@main struct Main { + @MainActor static func main() { + guard ProcessInfo.processInfo.environment["CI"] == "true", ProcessInfo.processInfo.environment["GITHUB_ACTIONS"] == "true", CommandLine.arguments.count == 3 else { + fputs("Fixture only runs on disposable GitHub CI.\n", stderr); exit(1) + } + let app = NSApplication.shared + app.setActivationPolicy(.regular) + let fixture = Fixture(output: CommandLine.arguments[1], title: CommandLine.arguments[2]) + app.delegate = fixture + withExtendedLifetime(fixture) { app.run() } + } +} From 3fa5426e739745e73d2bd967836ac521cacf14db Mon Sep 17 00:00:00 2001 From: Swaraj Bachu Date: Sat, 3 Oct 2026 22:37:31 +0000 Subject: [PATCH 07/10] test: expose native failure stages in live Mac verification --- scripts/test-macos-desktop-live.ts | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/scripts/test-macos-desktop-live.ts b/scripts/test-macos-desktop-live.ts index 38732d0..2df8928 100644 --- a/scripts/test-macos-desktop-live.ts +++ b/scripts/test-macos-desktop-live.ts @@ -59,6 +59,7 @@ const pending = new Map< resolve(value: unknown): void; reject(error: Error): void; timer: ReturnType; + method: string; } >(); function disconnected() { @@ -79,7 +80,13 @@ lines.on("line", (line) => { pending.delete(reply.id); clearTimeout(task.timer); if (reply.ok) task.resolve(reply.data); - else task.reject(Object.assign(Error(reply.error.message), reply.error)); + else { + console.error( + "NATIVE FAILURE", + JSON.stringify({ method: task.method, ...reply.error }), + ); + task.reject(Object.assign(Error(reply.error.message), reply.error)); + } } catch (error) { disconnected(); driverProcess.kill(); @@ -95,7 +102,7 @@ const driver: MacDriver = { reject(Error(`Native ${method} timed out.`)); driverProcess.kill(); }, 15_000); - pending.set(id, { resolve, reject, timer }); + pending.set(id, { resolve, reject, timer, method }); driverProcess.stdin.write(JSON.stringify({ ...args, id, method }) + "\n"); }); signal?.throwIfAborted(); @@ -260,6 +267,9 @@ try { }), /lease|control/i, ); + console.log( + "PASS capture, overlay cursors, and lease takeover; next: native fixture focus.", + ); // Focus only our exact disposable window before delivering physical input. const observed = await c.observe(human, { kind: "display" }); await call(human, "input", { @@ -294,9 +304,12 @@ try { x: point.x - observation.desktopBounds.x, y: point.y - observation.desktopBounds.y, }); + console.log("PASS fixture focus; next: native field click."); await act(click(manifest.field)); const text = "Opcode shared desktop CI Unicode α🙂 and chunked text verified"; + console.log("PASS field click; next: native Unicode text."); await act({ kind: "text", text }); + console.log("PASS text dispatch; next: native save click."); await act(click(manifest.save)); const result = await poll(async () => { try { From 0cc79547185989b778129ae5a20b10120c1615cd Mon Sep 17 00:00:00 2001 From: Swaraj Bachu Date: Sat, 3 Oct 2026 22:40:55 +0000 Subject: [PATCH 08/10] fix: preserve Mac window origins for native input coordinates --- native/macos/Driver.swift | 9 ++++++--- scripts/test-macos-desktop-live.ts | 14 +++++++++++++- 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/native/macos/Driver.swift b/native/macos/Driver.swift index 391f1d5..ad111f0 100644 --- a/native/macos/Driver.swift +++ b/native/macos/Driver.swift @@ -607,7 +607,7 @@ struct SavedElement { (item[kCGWindowLayer as String] as? Int) == 0 else { return nil } let frame = rect.integral return ["id": id, "pid": pid, "title": item[kCGWindowName as String] as? String ?? "", - "x": frame.minX, "y": frame.minY, "width": Int(frame.width), "height": Int(frame.height)] + "x": Double(frame.minX), "y": Double(frame.minY), "width": Int(frame.width), "height": Int(frame.height)] } } @@ -617,7 +617,7 @@ struct SavedElement { } if kind == "display" { let id = CGMainDisplayID(), frame = CGDisplayBounds(id).integral - return ["id": Int(id), "x": frame.minX, "y": frame.minY, "width": Int(frame.width), "height": Int(frame.height)] + return ["id": Int(id), "x": Double(frame.minX), "y": Double(frame.minY), "width": Int(frame.width), "height": Int(frame.height)] } guard kind == "window", let id = target["id"] as? Int, let window = try desktopWindows().first(where: { $0["id"] as? Int == id }) else { @@ -704,7 +704,10 @@ struct SavedElement { } } } - let origin = CGPoint(x: geometry["x"] as? Double ?? 0, y: geometry["y"] as? Double ?? 0) + guard let originX = geometry["x"] as? Double, let originY = geometry["y"] as? Double else { + throw DriverFailure(code: "stale_observation", message: "Desktop coordinate origin is unavailable.") + } + let origin = CGPoint(x: originX, y: originY) let width = geometry["width"] as! Int, height = geometry["height"] as! Int if (request["target"] as? [String: Any])?["kind"] as? String == "window", ["scroll", "buttonDown"].contains(kind) { let bounds = CGRect(x: origin.x, y: origin.y, width: Double(width), height: Double(height)) diff --git a/scripts/test-macos-desktop-live.ts b/scripts/test-macos-desktop-live.ts index 2df8928..419bdd4 100644 --- a/scripts/test-macos-desktop-live.ts +++ b/scripts/test-macos-desktop-live.ts @@ -282,6 +282,11 @@ try { (await backend.state()).focus === window.id ? true : undefined, "Fixture window did not become focused.", ); + assert.notEqual( + window.x, + 0, + "Fixture must exercise a nonzero horizontal window origin.", + ); const target = { kind: "window" as const, id: window.id }; async function act(action: Action | ((observation: Observation) => Action)) { await delay(200); @@ -307,7 +312,14 @@ try { console.log("PASS fixture focus; next: native field click."); await act(click(manifest.field)); const text = "Opcode shared desktop CI Unicode α🙂 and chunked text verified"; - console.log("PASS field click; next: native Unicode text."); + await poll( + async () => + (await backend.state()).focus === window.id ? true : undefined, + "Native field click moved focus away from the fixture; check nonzero window-origin coordinate mapping.", + ); + console.log( + "PASS field click retained exact window focus; next: native Unicode text.", + ); await act({ kind: "text", text }); console.log("PASS text dispatch; next: native save click."); await act(click(manifest.save)); From a9bb575e80b2af5844826a8fe3e3f50fc2bd1cc5 Mon Sep 17 00:00:00 2001 From: Swaraj Bachu Date: Sun, 4 Oct 2026 06:23:47 +0000 Subject: [PATCH 09/10] fix: harden desktop credentials and Mac broker ownership --- .github/workflows/shared-desktop.yml | 3 + docs/MAC_DESKTOP.md | 7 ++ native/macos/Driver.swift | 35 +++++++++ src/desktop/cli.ts | 18 ++--- src/desktop/client.ts | 5 +- src/desktop/server.ts | 19 +++-- tests/desktop-client.test.ts | 30 +++++++ tests/mac-lock.test.ts | 113 +++++++++++++++++++++++++++ 8 files changed, 212 insertions(+), 18 deletions(-) create mode 100644 tests/mac-lock.test.ts diff --git a/.github/workflows/shared-desktop.yml b/.github/workflows/shared-desktop.yml index e3c4415..aa2e52e 100644 --- a/.github/workflows/shared-desktop.yml +++ b/.github/workflows/shared-desktop.yml @@ -19,6 +19,8 @@ jobs: timeout-minutes: 20 steps: - uses: actions/checkout@v4 + with: + persist-credentials: false - uses: oven-sh/setup-bun@v2 with: bun-version: 1.3.10 @@ -34,6 +36,7 @@ jobs: CU_NATIVE_DIR: ${{ runner.temp }}/opcode-native run: | bash scripts/build-native.sh + bun test tests/mac-lock.test.ts node scripts/test-macos-desktop.mjs bun scripts/test-macos-desktop-live.ts - run: bun test diff --git a/docs/MAC_DESKTOP.md b/docs/MAC_DESKTOP.md index 4a2ecda..d27b244 100644 --- a/docs/MAC_DESKTOP.md +++ b/docs/MAC_DESKTOP.md @@ -147,6 +147,13 @@ required. The service does not reconnect and replay a pending native action. It persists broker-held key/button state for release on startup. Broker shutdown does not terminate apps or the installed helper. +Mac broker ownership uses a private per-user file lock held by its native helper +connection. Disconnects and crashes release the kernel lock automatically; the +lock file remains in place and must not be deleted to recover a session. The +private broker descriptor also remains after shutdown so a retiring broker cannot +remove a replacement broker's descriptor. `attach` checks the recorded endpoint +and replaces a stale descriptor when it starts the next broker. + The viewer polls screenshots; this is not a video/audio streaming transport. Optional recordings require an installed FFmpeg with `libx264`; health reports whether recording is available. Capture can include private desktop contents, so diff --git a/native/macos/Driver.swift b/native/macos/Driver.swift index ad111f0..cae823e 100644 --- a/native/macos/Driver.swift +++ b/native/macos/Driver.swift @@ -1,4 +1,5 @@ import Foundation +import Darwin import AppKit import ApplicationServices import ScreenCaptureKit @@ -23,6 +24,39 @@ struct SavedElement { var lastCaptureImage: CGImage? var virtualCursor: CGPoint? var desktopHeldModifiers: CGEventFlags = [] + // Each persistent helper connection owns its own Driver and file descriptor. + // Keep the inode stable: kernel flock releases on disconnect/crash, without + // PID reuse checks or racing stale socket/file deletion. + private var desktopBrokerLock: Int32 = -1 + deinit { + if desktopBrokerLock >= 0 { Darwin.close(desktopBrokerLock) } + } + func acquireDesktopBrokerLock() throws -> [String: Any] { + if desktopBrokerLock >= 0 { return ["locked": true] } + let directory = "/tmp/opcode-cu-\(getuid())" + do { + try FileManager.default.createDirectory(atPath: directory, withIntermediateDirectories: false, attributes: [.posixPermissions: 0o700]) + } catch { /* Validate an existing private directory below. */ } + var directoryInfo = stat() + guard lstat(directory, &directoryInfo) == 0, directoryInfo.st_uid == getuid(), + (directoryInfo.st_mode & S_IFMT) == S_IFDIR, (directoryInfo.st_mode & 0o077) == 0 else { + throw DriverFailure(code: "permission_denied", message: "Unsafe desktop lock directory.") + } + let fd = Darwin.open(directory + "/desktop-broker.lock", O_CREAT | O_RDWR | O_NOFOLLOW | O_CLOEXEC, 0o600) + guard fd >= 0 else { throw DriverFailure(code: "permission_denied", message: "Cannot open desktop broker lock.") } + var info = stat() + guard fstat(fd, &info) == 0, info.st_uid == getuid(), (info.st_mode & S_IFMT) == S_IFREG, + (info.st_mode & 0o077) == 0, info.st_nlink == 1 else { + Darwin.close(fd) + throw DriverFailure(code: "permission_denied", message: "Unsafe desktop broker lock file.") + } + guard flock(fd, LOCK_EX | LOCK_NB) == 0 else { + Darwin.close(fd) + throw DriverFailure(code: "lease_conflict", message: "A broker already owns this display. Reuse its endpoint.") + } + desktopBrokerLock = fd + return ["locked": true] + } let desktopKeyCodes: [String: CGKeyCode] = ["a":0,"s":1,"d":2,"f":3,"h":4,"g":5,"z":6,"x":7,"c":8,"v":9,"b":11,"q":12,"w":13,"e":14,"r":15,"y":16,"t":17,"1":18,"2":19,"3":20,"4":21,"6":22,"5":23,"9":25,"7":26,"8":28,"0":29,"o":31,"u":32,"i":34,"p":35,"l":37,"j":38,"k":40,"n":45,"m":46,"Enter":36,"Tab":48,"Space":49,"Backspace":51,"Escape":53,"Meta":55,"Shift":56,"Alt":58,"Control":59,"Home":115,"PageUp":116,"Delete":117,"End":119,"PageDown":121,"ArrowLeft":123,"ArrowRight":124,"ArrowDown":125,"ArrowUp":126,"F1":122,"F2":120,"F3":99,"F4":118,"F5":96,"F6":97,"F7":98,"F8":100,"F9":101,"F10":109,"F11":103,"F12":111] let foregroundAllowed = ProcessInfo.processInfo.environment["JEV_INTERACTION_MODE"] == "foreground" var activatedRenderers = Set() @@ -812,6 +846,7 @@ struct SavedElement { func handle(_ request: [String: Any]) async throws -> Any { switch request["method"] as? String { case "desktopKeys": return desktopKeyCodes.keys.sorted() + case "desktopLock": return try acquireDesktopBrokerLock() case "desktopGeometry": return try desktopGeometry(request) case "desktopWindows": return try desktopWindows() case "desktopState": return desktopState() diff --git a/src/desktop/cli.ts b/src/desktop/cli.ts index 3a386f3..5b4cf15 100644 --- a/src/desktop/cli.ts +++ b/src/desktop/cli.ts @@ -295,14 +295,12 @@ serve owns only the broker; exit never destroys an attached display.`); "Use a scoped --credential-file created by share, never the host descriptor.", ); const d = await descriptor(); - const sessionResponse = await fetch( - d.endpoint.replace(/\/$/, "") + "/session", - { - headers: { Authorization: "Bearer " + d.token }, - redirect: "error", - signal: AbortSignal.timeout(10000), - }, - ); + const viewerBase = serviceURL(d.endpoint).href.replace(/\/$/, ""); + const sessionResponse = await fetch(viewerBase + "/session", { + headers: { Authorization: "Bearer " + d.token }, + redirect: "error", + signal: AbortSignal.timeout(10000), + }); const session = (await sessionResponse.json()) as { admin?: boolean; scopes?: string[]; @@ -317,9 +315,7 @@ serve owns only the broker; exit never destroys an attached display.`); "A scoped viewer credential is required.", ); await desktopCall(d, "presence.list", {}); - console.log( - d.endpoint.replace(/\/$/, "") + "/view#" + encodeURIComponent(d.token), - ); + console.log(viewerBase + "/view#" + encodeURIComponent(d.token)); } else if (command === "call") { if (!values.method) throw failure("invalid_request", "Supply --method."); console.log( diff --git a/src/desktop/client.ts b/src/desktop/client.ts index 65208d2..e7fe29d 100644 --- a/src/desktop/client.ts +++ b/src/desktop/client.ts @@ -31,7 +31,10 @@ export function serviceURL(endpoint: string) { } export async function readCredential(path: string): Promise { - const file = await open(path, constants.O_RDONLY | constants.O_NOFOLLOW); + const file = await open( + path, + constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK, + ); try { const info = await file.stat(); if ( diff --git a/src/desktop/server.ts b/src/desktop/server.ts index 4deb0ac..fbbc888 100644 --- a/src/desktop/server.ts +++ b/src/desktop/server.ts @@ -68,9 +68,9 @@ export async function startDesktop(input: z.input) { ), ), ); - // A loopback reservation avoids stale filesystem sockets after a Mac crash. - // Linux retains its namespace-wide abstract X11 display reservation. - if (mac) lock.listen(49152 + (config.uid % 16000), "127.0.0.1", resolve); + // Linux keeps its namespace-wide abstract X11 reservation. Mac ownership is + // acquired below through the persistent native helper connection and flock. + if (mac) resolve(); else lock.listen( "\0opcode-desktop-" + @@ -99,6 +99,9 @@ export async function startDesktop(input: z.input) { const macDriver = mac ? await connectMacDriver() : undefined; startupDriver = macDriver; if (macDriver) { + z.object({ locked: z.literal(true) }).parse( + await macDriver.request("desktopLock"), + ); const status = z .object({ accessibility: z.boolean(), screenRecording: z.boolean() }) .parse(await macDriver.request("status")); @@ -388,21 +391,25 @@ export async function startDesktop(input: z.input) { if (closed) return; closed = true; await controller.close(); - macDriver?.close?.(); server.closeAllConnections(); await new Promise((resolve) => server.close(() => resolve())); - await unlink(path).catch(() => {}); + // A lost Mac helper connection can release flock before HTTP teardown. + // Never unlink its descriptor here: a successor may already have replaced + // it. A stale private descriptor is probed on attach and replaced by the + // next owner, exactly as after an abrupt process exit. + if (!mac) await unlink(path).catch(() => {}); + macDriver?.close?.(); lock.close(); }; return { controller, endpoint: descriptor.endpoint, path, close }; } catch (error) { await startupController?.close().catch(() => {}); - startupDriver?.close?.(); startupServer?.closeAllConnections(); if (startupServer?.listening) await new Promise((resolve) => startupServer!.close(() => resolve()), ); + startupDriver?.close?.(); lock.close(); throw error; } diff --git a/tests/desktop-client.test.ts b/tests/desktop-client.test.ts index 7431ea8..2a77b19 100644 --- a/tests/desktop-client.test.ts +++ b/tests/desktop-client.test.ts @@ -1,4 +1,5 @@ import assert from "node:assert/strict"; +import { execFileSync, spawnSync } from "node:child_process"; import { chmod, mkdtemp, readFile, rm, symlink } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; @@ -94,3 +95,32 @@ test("remote calls bind generation, disable redirects and never retry uncertain (error: any) => error.delivery === "unknown", ); }); + +test("credential named pipes are rejected without waiting for a writer", async () => { + const dir = await mkdtemp(join(tmpdir(), "cu-credential-fifo-")); + try { + const path = join(dir, "credential.fifo"); + execFileSync("mkfifo", ["-m", "600", path]); + // Isolate the reader so a blocking-open regression cannot hang the suite. + const result = spawnSync( + process.execPath, + [ + "--eval", + ` + import { readCredential } from ${JSON.stringify(new URL("../src/desktop/client.ts", import.meta.url).href)}; + try { + await readCredential(${JSON.stringify(path)}); + process.exit(2); + } catch (error) { + process.exit(error.code === "permission_denied" ? 0 : 3); + } + `, + ], + { timeout: 3000, encoding: "utf8" }, + ); + assert.equal(result.error, undefined, result.error?.message); + assert.equal(result.status, 0, result.stderr); + } finally { + await rm(dir, { recursive: true, force: true }); + } +}); diff --git a/tests/mac-lock.test.ts b/tests/mac-lock.test.ts new file mode 100644 index 0000000..45a69d9 --- /dev/null +++ b/tests/mac-lock.test.ts @@ -0,0 +1,113 @@ +import assert from "node:assert/strict"; +import { spawn } from "node:child_process"; +import { once } from "node:events"; +import { stat } from "node:fs/promises"; +import { join } from "node:path"; +import { test } from "node:test"; +import { setTimeout as delay } from "node:timers/promises"; +import type { MacDriver } from "../src/desktop/mac-backend.js"; +import { connectMacDriver } from "../src/desktop/mac-driver.js"; + +// This starts our native helper only on disposable Mac CI; no screen/input +// permissions are needed. Normal developer test runs never touch their helper. +const native = process.env.CU_NATIVE_DIR; +const enabled = + process.platform === "darwin" && + process.env.CI === "true" && + process.env.GITHUB_ACTIONS === "true" && + Boolean(native); +(enabled ? test : test.skip)( + "native broker flock excludes concurrent connections and recovers from disconnect/crash on a stable inode", + { + timeout: 30000, + }, + async () => { + let helper: ReturnType | undefined; + const connections: MacDriver[] = []; + const start = async () => { + helper = spawn(join(native!, "cu-helper"), [], { stdio: "ignore" }); + helper.on("error", () => {}); + for (let i = 0; i < 100; i++) { + if (helper.exitCode !== null || helper.signalCode !== null) + throw Error("Temporary native helper exited before startup."); + try { + const driver = await connectMacDriver(); + connections.push(driver); + return driver; + } catch { + await delay(50); + } + } + throw Error("Temporary native helper failed to start."); + }; + const acquire = async (driver: MacDriver) => { + for (let i = 0; i < 30; i++) { + try { + return await driver.request("desktopLock"); + } catch (error) { + if ((error as { code?: string }).code !== "DriverBusy") throw error; + await delay(10); + } + } + throw Error("Native lock remained busy."); + }; + try { + const first = await start(); + for (let i = 0; i < 7; i++) connections.push(await connectMacDriver()); + const results = await Promise.allSettled(connections.map(acquire)); + const winners = results.flatMap((result, index) => + result.status === "fulfilled" ? [index] : [], + ); + assert.equal(winners.length, 1); + for (const result of results) + if (result.status === "rejected") + assert.equal(result.reason.code, "lease_conflict"); + const winner = connections[winners[0]!]!; + assert.deepEqual(await acquire(winner), { locked: true }); + const path = `/tmp/opcode-cu-${process.getuid!()}/desktop-broker.lock`; + const original = await stat(path); + assert.equal(original.mode & 0o077, 0); + assert.equal(original.uid, process.getuid!()); + winner.close?.(); + const successor = connections.find((driver) => driver !== winner)!; + let recovered = false; + for (let i = 0; i < 100; i++) { + try { + assert.deepEqual(await acquire(successor), { locked: true }); + recovered = true; + break; + } catch (error) { + if ((error as { code?: string }).code !== "lease_conflict") + throw error; + await delay(20); + } + } + assert.equal( + recovered, + true, + "Disconnect must dispose the Driver's lock descriptor.", + ); + assert.equal((await stat(path)).ino, original.ino); + const exited = once(helper!, "exit"); + helper!.kill("SIGKILL"); + await exited; + for (const driver of connections) driver.close?.(); + connections.length = 0; + const restarted = await start(); + assert.deepEqual(await acquire(restarted), { locked: true }); + assert.equal( + (await stat(path)).ino, + original.ino, + "Crash recovery must reuse the stable lock file, never unlink it.", + ); + assert.ok(first); + } finally { + for (const driver of connections) driver.close?.(); + if (helper && helper.exitCode === null && helper.signalCode === null) { + const exited = once(helper, "exit"); + helper.kill("SIGTERM"); + await exited; + } + } + }, +); From 85c0c030e854ff45950ce1dad4b5d9feac54bc81 Mon Sep 17 00:00:00 2001 From: Swaraj Bachu Date: Sun, 4 Oct 2026 06:28:15 +0000 Subject: [PATCH 10/10] fix: keep Mac helper socket I/O off the cooperative executor --- native/macos/Helper.swift | 64 ++++++++++++++++++++++++++++----------- tests/mac-lock.test.ts | 21 +++++++++++++ 2 files changed, 68 insertions(+), 17 deletions(-) diff --git a/native/macos/Helper.swift b/native/macos/Helper.swift index de9e9f0..44ec9ce 100644 --- a/native/macos/Helper.swift +++ b/native/macos/Helper.swift @@ -8,6 +8,40 @@ import ApplicationServices @main struct HelperMain { @MainActor static var busy = false @MainActor static var guide: Process? + @MainActor static var activeClients = 0 + // Blocking sockets must not occupy Swift's cooperative executor: enough + // idle peers would otherwise prevent every pending reply from resuming. + // Admission caps this queue at 32 client operations plus one accept. + static let socketIO = DispatchQueue(label: "com.opcodehq.helper.socket-io", qos: .userInitiated, attributes: .concurrent) + static func blockingIO(_ operation: @escaping () -> T) async -> T { + await withCheckedContinuation { continuation in + socketIO.async { continuation.resume(returning: operation()) } + } + } + static func readChunk(_ client: Int32) async -> Data? { + await blockingIO { + var bytes = [UInt8](repeating: 0, count: 8192) + while true { + let count = Darwin.read(client, &bytes, bytes.count) + if count < 0 && errno == EINTR { continue } + return count > 0 ? Data(bytes.prefix(count)) : nil + } + } + } + static func writeReply(_ data: Data, client: Int32) async -> Bool { + await blockingIO { + data.withUnsafeBytes { raw -> Bool in + var offset = 0 + while offset < raw.count { + let count = Darwin.write(client, raw.baseAddress!.advanced(by: offset), raw.count - offset) + if count < 0 && errno == EINTR { continue } + if count <= 0 { return false } + offset += count + } + return true + } + } + } @MainActor static func showPermissionGuide(_ kind: String) { if guide?.isRunning == true { guide?.terminate() } let process = Process() @@ -90,32 +124,28 @@ import ApplicationServices } guard bound == 0, chmod(path, 0o600) == 0, listen(server, 16) == 0 else { exit(1) } while true { - let client = await Task.detached { accept(server, nil, nil) }.value + let client = await blockingIO { Darwin.accept(server, nil, nil) } guard client >= 0 else { continue } var uid: uid_t = 0, gid: gid_t = 0 guard getpeereid(client, &uid, &gid) == 0, uid == getuid() else { Darwin.close(client); continue } + guard activeClients < 32 else { Darwin.close(client); continue } + // A peer that stops reading must not retain a reply thread forever. + var writeTimeout = timeval(tv_sec: 10, tv_usec: 0) + guard setsockopt(client, SOL_SOCKET, SO_SNDTIMEO, &writeTimeout, socklen_t(MemoryLayout.size)) == 0 else { + Darwin.close(client); continue + } + activeClients += 1 let driver = Driver() - Task.detached { - defer { Darwin.close(client) } + Task { @MainActor in + defer { Darwin.close(client); activeClients -= 1 } var buffer = Data() - var chunk = [UInt8](repeating: 0, count: 8192) - while true { - let count = Darwin.read(client, &chunk, chunk.count) - if count <= 0 { break } - buffer.append(contentsOf: chunk.prefix(count)) + while let chunk = await readChunk(client) { + buffer.append(chunk) while let newline = buffer.firstIndex(of: 10) { let line = Data(buffer[.. 128_000 { return } let data = await reply(line, driver: driver) - let sent = data.withUnsafeBytes { raw -> Bool in - var offset = 0 - while offset < raw.count { - let count = Darwin.write(client, raw.baseAddress!.advanced(by: offset), raw.count - offset) - if count <= 0 { return false } - offset += count - } - return true - } + let sent = await writeReply(data, client: client) if !sent { return } } if buffer.count > 128_000 { break } diff --git a/tests/mac-lock.test.ts b/tests/mac-lock.test.ts index 45a69d9..e459409 100644 --- a/tests/mac-lock.test.ts +++ b/tests/mac-lock.test.ts @@ -54,6 +54,27 @@ const enabled = try { const first = await start(); for (let i = 0; i < 7; i++) connections.push(await connectMacDriver()); + // Seven idle sockets must not occupy the executor needed for this reply. + let idleDeadline: ReturnType | undefined; + try { + assert.deepEqual( + await Promise.race([ + first.request("cancel"), + new Promise((_, reject) => { + idleDeadline = setTimeout( + () => + reject( + Error("Idle native sockets starved an active request."), + ), + 3000, + ); + }), + ]), + { cancelled: true }, + ); + } finally { + clearTimeout(idleDeadline); + } const results = await Promise.allSettled(connections.map(acquire)); const winners = results.flatMap((result, index) => result.status === "fulfilled" ? [index] : [],