diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3a674be92..d34597a53 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -5,6 +5,7 @@ on: branches: - main - 'release/**' + - 'feat/computer-key-chords' pull_request: types: [opened, synchronize, reopened, ready_for_review, labeled, unlabeled] merge_group: diff --git a/crates/webcodex-computer/src/lib.rs b/crates/webcodex-computer/src/lib.rs index 880cf7f30..11557bccb 100644 --- a/crates/webcodex-computer/src/lib.rs +++ b/crates/webcodex-computer/src/lib.rs @@ -2996,12 +2996,12 @@ mod key_input_runtime_tests { .iter() .map(|modifier| modifier.to_string()) .collect::>(); - for key in COMPUTER_KEY_INPUT_KEYS { + for key in webcodex_core::computer_input_contract::computer_key_input_keys() { assert!(validate_key_input(key, &[]).is_ok()); assert!(validate_key_input(key, &modifiers).is_ok()); } assert!(validate_key_input("tab", &["shift".to_string()]).is_ok()); - assert!(validate_key_input("a", &[]).is_err()); + assert!(validate_key_input("unknown", &[]).is_err()); assert!(validate_key_input("enter", &["shift".to_string(), "shift".to_string()]).is_err()); } } diff --git a/crates/webcodex-computer/src/platform/macos/input.rs b/crates/webcodex-computer/src/platform/macos/input.rs index 0690a4e25..0889ff5d9 100644 --- a/crates/webcodex-computer/src/platform/macos/input.rs +++ b/crates/webcodex-computer/src/platform/macos/input.rs @@ -1350,9 +1350,9 @@ pub(crate) fn key_input( modifiers: &[String], ) -> Result { validate_key_input(key, modifiers)?; - if webcodex_core::computer_input_contract::is_computer_letter_shortcut(key) { + if !webcodex_core::computer_input_contract::COMPUTER_KEY_INPUT_KEYS.contains(&key) { return Err( - "invalid_request: computer letter shortcuts are supported only on Windows".into(), + "invalid_request: extended computer key chords are supported only on Windows".into(), ); } if !unsafe { AXIsProcessTrusted() } { diff --git a/crates/webcodex-computer/src/platform/windows/input.rs b/crates/webcodex-computer/src/platform/windows/input.rs index a0cdc9910..86d7d4626 100644 --- a/crates/webcodex-computer/src/platform/windows/input.rs +++ b/crates/webcodex-computer/src/platform/windows/input.rs @@ -333,8 +333,51 @@ fn windows_key_mapping(key: &str) -> Result<(VIRTUAL_KEY, bool), String> { "page_down" => Ok((VK_NEXT, true)), "home" => Ok((VK_HOME, true)), "end" => Ok((VK_END, true)), - "j" => Ok((VIRTUAL_KEY(0x4A), false)), - "p" => Ok((VIRTUAL_KEY(0x50), false)), + "space" => Ok((VK_SPACE, false)), + "backspace" => Ok((VK_BACK, false)), + "delete" => Ok((VK_DELETE, true)), + "insert" => Ok((VK_INSERT, true)), + "semicolon" => Ok((VK_OEM_1, false)), + "equal" => Ok((VK_OEM_PLUS, false)), + "comma" => Ok((VK_OEM_COMMA, false)), + "minus" => Ok((VK_OEM_MINUS, false)), + "period" => Ok((VK_OEM_PERIOD, false)), + "slash" => Ok((VK_OEM_2, false)), + "backquote" => Ok((VK_OEM_3, false)), + "left_bracket" => Ok((VK_OEM_4, false)), + "backslash" => Ok((VK_OEM_5, false)), + "right_bracket" => Ok((VK_OEM_6, false)), + "quote" => Ok((VK_OEM_7, false)), + "numpad_add" => Ok((VK_ADD, false)), + "numpad_subtract" => Ok((VK_SUBTRACT, false)), + "numpad_multiply" => Ok((VK_MULTIPLY, false)), + "numpad_divide" => Ok((VK_DIVIDE, true)), + "numpad_decimal" => Ok((VK_DECIMAL, false)), + "numpad_enter" => Ok((VK_RETURN, true)), + _ if key.len() == 1 && key.as_bytes()[0].is_ascii_lowercase() => Ok(( + VIRTUAL_KEY(u16::from(key.as_bytes()[0].to_ascii_uppercase())), + false, + )), + _ if key.len() == 1 && key.as_bytes()[0].is_ascii_digit() => { + Ok((VIRTUAL_KEY(u16::from(key.as_bytes()[0])), false)) + } + _ if key + .strip_prefix('f') + .and_then(|n| n.parse::().ok()) + .is_some_and(|n| (1..=24).contains(&n)) => + { + let number = key[1..].parse::().expect("function key range checked"); + Ok((VIRTUAL_KEY(VK_F1.0 + number - 1), false)) + } + _ if key + .strip_prefix("numpad_") + .is_some_and(|n| n.len() == 1 && n.as_bytes()[0].is_ascii_digit()) => + { + Ok(( + VIRTUAL_KEY(VK_NUMPAD0.0 + u16::from(key.as_bytes()[7] - b'0')), + false, + )) + } _ => Err("invalid_request: computer key is outside the closed vocabulary".to_string()), } } @@ -357,8 +400,9 @@ fn windows_modifier_key(modifier: &str) -> Result { fn validate_windows_key_input_chord(key: &str, modifiers: &[String]) -> Result<(), String> { let has_option = modifiers.iter().any(|modifier| modifier == "option"); let has_control = modifiers.iter().any(|modifier| modifier == "control"); - let escapes_exact_surface = - (has_option && matches!(key, "tab" | "escape")) || (has_control && key == "escape"); + let escapes_exact_surface = (has_option && matches!(key, "tab" | "escape")) + || (has_control && key == "escape") + || (has_control && has_option && key == "delete"); if escapes_exact_surface { return Err( "key_input_failed: Windows system-level key chord is outside the exact-surface input contract" diff --git a/crates/webcodex-computer/src/platform/windows/mod.rs b/crates/webcodex-computer/src/platform/windows/mod.rs index 0421e123c..4bde2a4af 100644 --- a/crates/webcodex-computer/src/platform/windows/mod.rs +++ b/crates/webcodex-computer/src/platform/windows/mod.rs @@ -75,10 +75,13 @@ use windows::{ GetAsyncKeyState, SendInput, INPUT, INPUT_0, INPUT_KEYBOARD, INPUT_MOUSE, KEYBDINPUT, KEYBD_EVENT_FLAGS, KEYEVENTF_EXTENDEDKEY, KEYEVENTF_KEYUP, MOUSEEVENTF_ABSOLUTE, MOUSEEVENTF_LEFTDOWN, MOUSEEVENTF_LEFTUP, MOUSEEVENTF_MOVE, MOUSEEVENTF_VIRTUALDESK, - MOUSEINPUT, MOUSE_EVENT_FLAGS, VIRTUAL_KEY, VK_CONTROL, VK_DOWN, VK_END, VK_ESCAPE, - VK_HOME, VK_LBUTTON, VK_LCONTROL, VK_LEFT, VK_LMENU, VK_LSHIFT, VK_LWIN, VK_MBUTTON, - VK_MENU, VK_NEXT, VK_PRIOR, VK_RBUTTON, VK_RCONTROL, VK_RETURN, VK_RIGHT, VK_RMENU, - VK_RSHIFT, VK_RWIN, VK_SHIFT, VK_TAB, VK_UP, VK_XBUTTON1, VK_XBUTTON2, + MOUSEINPUT, MOUSE_EVENT_FLAGS, VIRTUAL_KEY, VK_ADD, VK_BACK, VK_CONTROL, VK_DECIMAL, + VK_DELETE, VK_DIVIDE, VK_DOWN, VK_END, VK_ESCAPE, VK_F1, VK_HOME, VK_INSERT, + VK_LBUTTON, VK_LCONTROL, VK_LEFT, VK_LMENU, VK_LSHIFT, VK_LWIN, VK_MBUTTON, VK_MENU, + VK_MULTIPLY, VK_NEXT, VK_NUMPAD0, VK_OEM_1, VK_OEM_2, VK_OEM_3, VK_OEM_4, VK_OEM_5, + VK_OEM_6, VK_OEM_7, VK_OEM_COMMA, VK_OEM_MINUS, VK_OEM_PERIOD, VK_OEM_PLUS, VK_PRIOR, + VK_RBUTTON, VK_RCONTROL, VK_RETURN, VK_RIGHT, VK_RMENU, VK_RSHIFT, VK_RWIN, VK_SHIFT, + VK_SPACE, VK_SUBTRACT, VK_TAB, VK_UP, VK_XBUTTON1, VK_XBUTTON2, }, UI::Shell::{ Common::ITEMIDLIST, FOLDERID_AppsFolder, IEnumIDList, ILCombine, ILGetSize, diff --git a/crates/webcodex-computer/src/windows_uia_tests.rs b/crates/webcodex-computer/src/windows_uia_tests.rs index 099165898..9a0d8b9e4 100644 --- a/crates/webcodex-computer/src/windows_uia_tests.rs +++ b/crates/webcodex-computer/src/windows_uia_tests.rs @@ -718,7 +718,7 @@ fn computer_windows_key_input_native_plan_is_closed_and_releases_modifiers_in_re "{key}" ); } - assert!(platform::test_windows_key_input_plan("a", &[]).is_err()); + assert!(platform::test_windows_key_input_plan("unknown", &[]).is_err()); let option = platform::test_windows_key_input_plan("arrow_left", &["option".to_string()]) .expect("Windows option modifier must map to Alt for exact-surface chords"); @@ -791,7 +791,12 @@ fn computer_windows_key_input_letter_chords_have_complete_bounded_sequences() { vec!["control".into(), "shift".into()], vec!["control".into(), "option".into()], ] { - assert!(platform::test_windows_key_input_plan(key, &modifiers).is_err()); + assert_eq!( + platform::test_windows_key_input_plan(key, &modifiers).is_ok(), + !modifiers + .iter() + .any(|modifier: &String| modifier == "command") + ); } for held in [vk, 0xA0, 0xA1, 0xA2, 0xA3, 0xA4, 0xA5, 0x5B, 0x5C] { assert!(platform::test_windows_keyboard_state_guard(key, Some(held)).is_err()); @@ -807,6 +812,85 @@ fn computer_windows_key_input_letter_chords_have_complete_bounded_sequences() { } } +#[test] +fn computer_windows_general_chords_have_bounded_balanced_native_plans() { + for key in webcodex_core::computer_input_contract::computer_key_input_keys() { + for mask in 0..8 { + let modifiers: Vec<_> = ["shift", "control", "option"] + .iter() + .enumerate() + .filter(|(index, _)| mask & (1 << index) != 0) + .map(|(_, name)| (*name).to_string()) + .collect(); + let blocked = (modifiers.iter().any(|m| m == "option") + && matches!(key, "tab" | "escape")) + || (modifiers.iter().any(|m| m == "control") && key == "escape") + || (key == "delete" + && modifiers.iter().any(|m| m == "control") + && modifiers.iter().any(|m| m == "option")); + let result = platform::test_windows_key_input_plan(key, &modifiers); + if blocked { + assert!(result.unwrap_err().starts_with("key_input_failed:")); + continue; + } + let plan = result.unwrap_or_else(|error| panic!("{key} {modifiers:?}: {error}")); + assert_eq!(plan.len(), 2 * modifiers.len() + 2); + assert!(plan.len() <= 8); + for index in 0..plan.len() / 2 { + let down = plan[index]; + let up = plan[plan.len() - 1 - index]; + assert!(!down.1); + assert!(up.1); + assert_eq!((down.0, down.2), (up.0, up.2), "{key} {modifiers:?}"); + } + } + let native_key = platform::test_windows_key_input_plan(key, &[]).unwrap()[0].0; + assert!(platform::test_windows_keyboard_state_guard(key, Some(native_key)).is_err()); + } + for (key, vk, extended) in [ + ("a", 0x41, false), + ("z", 0x5A, false), + ("0", 0x30, false), + ("9", 0x39, false), + ("f1", 0x70, false), + ("f24", 0x87, false), + ("space", 0x20, false), + ("backspace", 0x08, false), + ("delete", 0x2E, true), + ("insert", 0x2D, true), + ("semicolon", 0xBA, false), + ("equal", 0xBB, false), + ("comma", 0xBC, false), + ("minus", 0xBD, false), + ("period", 0xBE, false), + ("slash", 0xBF, false), + ("backquote", 0xC0, false), + ("left_bracket", 0xDB, false), + ("backslash", 0xDC, false), + ("right_bracket", 0xDD, false), + ("quote", 0xDE, false), + ("numpad_0", 0x60, false), + ("numpad_9", 0x69, false), + ("numpad_add", 0x6B, false), + ("numpad_subtract", 0x6D, false), + ("numpad_multiply", 0x6A, false), + ("numpad_divide", 0x6F, true), + ("numpad_decimal", 0x6E, false), + ("numpad_enter", 0x0D, true), + ] { + assert_eq!( + platform::test_windows_key_input_plan(key, &[]).unwrap(), + vec![(vk, false, extended), (vk, true, extended)], + "{key}" + ); + } + for inserted in 1..8 { + assert!(platform::test_windows_send_input_count(inserted, 8) + .unwrap_err() + .starts_with("outcome_unknown:")); + } +} + #[test] fn computer_windows_key_input_zero_is_deterministic_and_partial_injection_is_unknown() { assert!(platform::test_windows_send_input_count(2, 2).is_ok()); diff --git a/crates/webcodex-core/src/computer_input_contract.rs b/crates/webcodex-core/src/computer_input_contract.rs index 192bb00fe..48736e509 100644 --- a/crates/webcodex-core/src/computer_input_contract.rs +++ b/crates/webcodex-core/src/computer_input_contract.rs @@ -22,6 +22,63 @@ pub const COMPUTER_KEY_INPUT_KEYS: &[&str] = &[ /// Windows virtual-key shortcuts. Each requires exactly the Control modifier. pub const COMPUTER_LETTER_SHORTCUT_KEYS: &[&str] = &["j", "p"]; +const LETTER_KEYS: &[&str] = &[ + "a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m", "n", "o", "p", "q", "r", "s", + "t", "u", "v", "w", "x", "y", "z", +]; +const DIGIT_KEYS: &[&str] = &["0", "1", "2", "3", "4", "5", "6", "7", "8", "9"]; +const FUNCTION_KEYS: &[&str] = &[ + "f1", "f2", "f3", "f4", "f5", "f6", "f7", "f8", "f9", "f10", "f11", "f12", "f13", "f14", "f15", + "f16", "f17", "f18", "f19", "f20", "f21", "f22", "f23", "f24", +]; +const EDITING_KEYS: &[&str] = &["space", "backspace", "delete", "insert"]; +const PUNCTUATION_KEYS: &[&str] = &[ + "semicolon", + "equal", + "comma", + "minus", + "period", + "slash", + "backquote", + "left_bracket", + "backslash", + "right_bracket", + "quote", +]; +const NUMPAD_KEYS: &[&str] = &[ + "numpad_0", + "numpad_1", + "numpad_2", + "numpad_3", + "numpad_4", + "numpad_5", + "numpad_6", + "numpad_7", + "numpad_8", + "numpad_9", + "numpad_add", + "numpad_subtract", + "numpad_multiply", + "numpad_divide", + "numpad_decimal", + "numpad_enter", +]; + +/// Logical virtual-key names, not Unicode text or physical scan-code positions. +pub fn computer_key_input_keys() -> impl Iterator { + [ + COMPUTER_KEY_INPUT_KEYS, + LETTER_KEYS, + DIGIT_KEYS, + FUNCTION_KEYS, + EDITING_KEYS, + PUNCTUATION_KEYS, + NUMPAD_KEYS, + ] + .into_iter() + .flat_map(|keys| keys.iter().copied()) +} + /// Canonical modifier order; platform adapters may reject unsupported chords. pub const COMPUTER_KEY_INPUT_MODIFIERS: &[&str] = &["shift", "control", "option", "command"]; @@ -30,14 +87,10 @@ pub fn is_computer_letter_shortcut(key: &str) -> bool { } pub fn validate_computer_key_input(key: &str, modifiers: &[String]) -> Result<(), &'static str> { - if !COMPUTER_KEY_INPUT_KEYS.contains(&key) && !is_computer_letter_shortcut(key) { + if !computer_key_input_keys().any(|supported| supported == key) { return Err("computer key input key is outside the closed vocabulary"); } - validate_computer_key_modifiers(modifiers)?; - if is_computer_letter_shortcut(key) && (modifiers.len() != 1 || modifiers[0] != "control") { - return Err("computer letter shortcuts require exactly modifiers=[control]"); - } - Ok(()) + validate_computer_key_modifiers(modifiers) } pub fn validate_computer_key_modifiers(modifiers: &[String]) -> Result<(), &'static str> { @@ -54,14 +107,23 @@ pub fn validate_computer_key_modifiers(modifiers: &[String]) -> Result<(), &'sta Ok(()) } -pub fn computer_key_input_capabilities(key: &str) -> &'static [RunnerCapabilityId] { - if is_computer_letter_shortcut(key) { +pub fn computer_key_input_capabilities( + key: &str, + modifiers: &[String], + general_chords: bool, +) -> &'static [RunnerCapabilityId] { + if COMPUTER_KEY_INPUT_KEYS.contains(&key) { + &[RunnerCapabilityId::ComputerKeyInput] + } else if !general_chords && is_computer_letter_shortcut(key) && modifiers == ["control"] { &[ RunnerCapabilityId::ComputerKeyInput, RunnerCapabilityId::ComputerLetterShortcuts, ] } else { - &[RunnerCapabilityId::ComputerKeyInput] + &[ + RunnerCapabilityId::ComputerKeyInput, + RunnerCapabilityId::ComputerKeyChords, + ] } } @@ -81,32 +143,13 @@ impl ComputerKeyInputPayload { } pub fn computer_key_schema(_: &mut schemars::SchemaGenerator) -> schemars::Schema { - let keys: Vec<_> = COMPUTER_KEY_INPUT_KEYS - .iter() - .chain(COMPUTER_LETTER_SHORTCUT_KEYS) - .collect(); + let keys: Vec<_> = computer_key_input_keys().collect(); schemars::json_schema!({ "type": "string", "enum": keys, }) } -/// Applied to the key action object so schema validation checks the complete chord. -pub fn computer_key_chord_constraints() -> serde_json::Value { - serde_json::json!([{ - "anyOf": [ - {"properties": {"key": {"enum": COMPUTER_KEY_INPUT_KEYS}}}, - { - "required": ["modifiers"], - "properties": { - "key": {"enum": COMPUTER_LETTER_SHORTCUT_KEYS}, - "modifiers": {"const": ["control"]} - } - } - ] - }]) -} - /// Preserve explicit null through the OpenAPI input-schema normalization path. pub fn computer_key_modifiers_schema(_: &mut schemars::SchemaGenerator) -> schemars::Schema { schemars::json_schema!({ @@ -124,3 +167,84 @@ pub fn computer_key_modifiers_schema(_: &mut schemars::SchemaGenerator) -> schem ] }) } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn computer_key_chord_vocabulary_and_modifier_sets_are_bounded() { + let keys: Vec<_> = computer_key_input_keys().collect(); + assert_eq!(keys.len(), 102); + assert_eq!( + keys.iter().collect::>().len(), + keys.len() + ); + for key in &keys { + for mask in 0..16 { + let modifiers: Vec<_> = COMPUTER_KEY_INPUT_MODIFIERS + .iter() + .enumerate() + .filter(|(index, _)| mask & (1 << index) != 0) + .map(|(_, name)| (*name).to_string()) + .collect(); + validate_computer_key_input(key, &modifiers).unwrap(); + } + } + for key in [ + "", + "A", + "Ctrl+P", + "f0", + "f01", + "f25", + "numpad_00", + "windows", + "a b", + ] { + assert!(validate_computer_key_input(key, &[]).is_err(), "{key}"); + } + for modifiers in [ + vec!["ctrl".into()], + vec!["control".into(), "control".into()], + vec!["shift".into(); 5], + ] { + assert!(validate_computer_key_input("a", &modifiers).is_err()); + } + } + + #[test] + fn computer_key_chord_capabilities_classify_the_complete_chord() { + use RunnerCapabilityId::{ComputerKeyChords, ComputerKeyInput, ComputerLetterShortcuts}; + assert_eq!( + computer_key_input_capabilities("tab", &[], false), + &[ComputerKeyInput] + ); + for key in ["j", "p"] { + assert_eq!( + computer_key_input_capabilities(key, &["control".into()], false), + &[ComputerKeyInput, ComputerLetterShortcuts] + ); + assert_eq!( + computer_key_input_capabilities(key, &["control".into()], true), + &[ComputerKeyInput, ComputerKeyChords] + ); + for modifiers in [ + vec![], + vec!["shift".into()], + vec!["control".into(), "shift".into()], + ] { + assert_eq!( + computer_key_input_capabilities(key, &modifiers, false), + &[ComputerKeyInput, ComputerKeyChords] + ); + } + } + for key in ["a", "0", "f24", "slash", "numpad_enter", "delete"] { + assert_eq!( + computer_key_input_capabilities(key, &["control".into()], true), + &[ComputerKeyInput, ComputerKeyChords] + ); + } + } +} diff --git a/crates/webcodex-core/src/runner_protocol.rs b/crates/webcodex-core/src/runner_protocol.rs index be8d35026..c3d5baa22 100644 --- a/crates/webcodex-core/src/runner_protocol.rs +++ b/crates/webcodex-core/src/runner_protocol.rs @@ -1082,6 +1082,13 @@ runner_capabilities! { #[serde(default, skip_serializing_if = "is_false")] pub computer_letter_shortcuts: bool = false; } + /// General Windows single-key chords; neither navigation nor the legacy + /// Ctrl+J/Ctrl+P capability implies this larger vocabulary or chord set. + ComputerKeyChords => RUNNER_CAPABILITY_COMPUTER_KEY_CHORDS("computer_key_chords"), + v2_baseline = false { + #[serde(default, skip_serializing_if = "is_false")] + pub computer_key_chords: bool = false; + } /// Native exact-window activation/raise. Missing on older Runners is false and /// is never inferred from accessibility control, observation, or platform. ComputerWindowActivate => RUNNER_CAPABILITY_COMPUTER_WINDOW_ACTIVATE("computer_window_activate"), @@ -2987,6 +2994,7 @@ mod envelope_tests { computer_scroll_to_element: false, computer_key_input: false, computer_letter_shortcuts: false, + computer_key_chords: false, computer_window_activate: false, computer_text_input: false, job_state_reconciliation: false, @@ -3348,6 +3356,24 @@ mod envelope_tests { ); } + #[test] + fn computer_key_chords_capability_is_not_implied_by_legacy_keys() { + let legacy: RunnerCapabilities = + serde_json::from_str(r#"{"computer_key_input":true,"computer_letter_shortcuts":true}"#) + .unwrap(); + assert!(!legacy.computer_key_chords); + let chords: RunnerCapabilities = + serde_json::from_str(r#"{"computer_key_chords":true}"#).unwrap(); + assert!(chords.supports(RunnerCapabilityId::ComputerKeyChords)); + assert!(!chords.computer_key_input); + assert!(!chords.computer_letter_shortcuts); + assert!(!RunnerCapabilities::default().computer_key_chords); + assert_eq!( + RunnerCapabilityId::from_wire_name("computer_key_chords"), + Some(RunnerCapabilityId::ComputerKeyChords) + ); + } + #[test] fn computer_window_activate_capability_deserializes_only_when_present() { let capabilities: RunnerCapabilities = diff --git a/crates/webcodex-runner-config/src/lib.rs b/crates/webcodex-runner-config/src/lib.rs index e58a2cf16..27f18dcfb 100644 --- a/crates/webcodex-runner-config/src/lib.rs +++ b/crates/webcodex-runner-config/src/lib.rs @@ -391,6 +391,7 @@ pub fn generated_runner_config_toml(opts: &RunnerInitOptions) -> Result bool { | RunnerFeature::ComputerScrollToElement | RunnerFeature::ComputerKeyInput | RunnerFeature::ComputerLetterShortcuts + | RunnerFeature::ComputerKeyChords | RunnerFeature::ComputerWindowActivate | RunnerFeature::ComputerTextInput ) diff --git a/crates/webcodex-runner-registry/src/dispatch_fence.rs b/crates/webcodex-runner-registry/src/dispatch_fence.rs index 30afe17fa..4261469fc 100644 --- a/crates/webcodex-runner-registry/src/dispatch_fence.rs +++ b/crates/webcodex-runner-registry/src/dispatch_fence.rs @@ -160,7 +160,7 @@ impl PendingDispatchFence { "invalid_request: invalid computer key input chord".into(), ); } - return webcodex_core::computer_input_contract::computer_key_input_capabilities(&input.key) + return webcodex_core::computer_input_contract::computer_key_input_capabilities(&input.key, &input.modifiers, runner.supports(RunnerFeature::ComputerKeyChords)) .iter().any(|feature| !runner.supports(*feature)) .then(|| "capability_unavailable: Computer key input capability changed before dispatch".into()); } diff --git a/crates/webcodex-runner-registry/src/requests.rs b/crates/webcodex-runner-registry/src/requests.rs index 80f8f8b06..048473950 100644 --- a/crates/webcodex-runner-registry/src/requests.rs +++ b/crates/webcodex-runner-registry/src/requests.rs @@ -2557,11 +2557,7 @@ impl RunnerRegistry { "computer_element_state" => &[RunnerFeature::ComputerElementState], "computer_control" => &[RunnerFeature::ComputerControl], "computer_scroll_to_element" => &[RunnerFeature::ComputerScrollToElement], - "computer_key_input" => { - webcodex_core::computer_input_contract::computer_key_input_capabilities( - &key_input.as_ref().expect("key payload validated").key, - ) - } + "computer_key_input" => &[RunnerFeature::ComputerKeyInput], "computer_pointer_move" | "computer_pointer_click" => { &[RunnerFeature::ComputerPointerControl] } @@ -2590,14 +2586,17 @@ impl RunnerRegistry { .get(&client_id) .ok_or_else(|| format!("unknown shell client: {client_id}"))?; assert_runner_access(auth, current)?; - let letter_instance = key_input + let required_features = key_input.as_ref().map_or(required_features, |input| { + webcodex_core::computer_input_contract::computer_key_input_capabilities( + &input.key, + &input.modifiers, + current.supports(RunnerFeature::ComputerKeyChords), + ) + }); + let key_instance = key_input .as_ref() - .filter(|input| { - webcodex_core::computer_input_contract::is_computer_letter_shortcut(&input.key) - }) .map(|_| current.runner_instance_id.clone()); - let expected_runner_instance_id = - expected_runner_instance_id.or(letter_instance.as_deref()); + let expected_runner_instance_id = expected_runner_instance_id.or(key_instance.as_deref()); if let Some(expected) = expected_runner_instance_id { if expected.is_empty() || current.runner_instance_id != expected { return Err( diff --git a/crates/webcodex-runner-registry/src/tests/computer_control.rs b/crates/webcodex-runner-registry/src/tests/computer_control.rs index 37fd58a3a..fbca8af3d 100644 --- a/crates/webcodex-runner-registry/src/tests/computer_control.rs +++ b/crates/webcodex-runner-registry/src/tests/computer_control.rs @@ -48,64 +48,135 @@ async fn computer_letter_shortcuts_require_both_capabilities_before_enqueue() { } #[tokio::test] -async fn computer_letter_shortcuts_recheck_access_and_capabilities_at_dequeue() { - for change in ["owner", "key_input", "letters", "instance"] { - let registry = RunnerRegistry::default(); - let mut registration = runner_registration("letters", "instance", Vec::new()); - registration.owner = Some("alice".into()); - registration.capabilities.computer_key_input = true; - registration.capabilities.computer_letter_shortcuts = true; - registry - .register(current_runner_registration(registration.clone())) - .await - .unwrap(); - let access = auth_context(Some("alice"), false); - let (_, rx) = registry +async fn computer_key_chords_recheck_access_and_capabilities_at_dequeue() { + for (key, modifiers, capability) in [ + ("j", vec!["control"], RunnerFeature::ComputerLetterShortcuts), + ( + "p", + vec!["control", "shift"], + RunnerFeature::ComputerKeyChords, + ), + ] { + for change in ["owner", "key_input", "extension", "instance"] { + let registry = RunnerRegistry::default(); + let mut registration = runner_registration("letters", "instance", Vec::new()); + registration.owner = Some("alice".into()); + registration.capabilities.computer_key_input = true; + registration.capabilities.set(capability, true); + registry + .register(current_runner_registration(registration.clone())) + .await + .unwrap(); + let access = auth_context(Some("alice"), false); + let (_, rx) = registry .enqueue_computer( "letters".into(), "computer_key_input", - r#"{"surface_id":"surface_test","key":"j","modifiers":["control"]}"#.into(), + serde_json::json!({"surface_id":"surface_test", "key":key, "modifiers":modifiers}).to_string(), "alice".into(), Some(&access), 5, ) .await .unwrap(); - match change { - "owner" => registration.owner = Some("bob".into()), - "key_input" => registration.capabilities.computer_key_input = false, - "instance" => registration.runner_instance_id = "replacement".into(), - _ => registration.capabilities.computer_letter_shortcuts = false, - } - registry - .register(current_runner_registration(registration)) - .await - .unwrap(); - assert!(registry - .poll(RunnerPollRequest { - client_id: "letters".into(), - runner_instance_id: if change == "instance" { - "replacement" - } else { - "instance" - } - .into() - }) - .await - .unwrap() - .is_none()); - let response = rx.await.unwrap(); - assert_eq!(response.request_dispatched, Some(false)); - assert!(response.error.unwrap().starts_with(match change { + match change { + "owner" => registration.owner = Some("bob".into()), + "key_input" => registration.capabilities.computer_key_input = false, + "instance" => registration.runner_instance_id = "replacement".into(), + _ => registration.capabilities.set(capability, false), + } + registry + .register(current_runner_registration(registration)) + .await + .unwrap(); + assert!(registry + .poll(RunnerPollRequest { + client_id: "letters".into(), + runner_instance_id: if change == "instance" { + "replacement" + } else { + "instance" + } + .into() + }) + .await + .unwrap() + .is_none()); + let response = rx.await.unwrap(); + assert_eq!(response.request_dispatched, Some(false)); + assert!(response.error.unwrap().starts_with(match change { "owner" => "client_access_denied:", "instance" => "runner instance was replaced before the request completed; the request was not handed to the replacement instance", _ => "capability_unavailable:", })); + } } } #[tokio::test] -async fn computer_letter_shortcuts_reject_stale_instances_and_invalid_chords() { +async fn computer_general_chords_require_the_new_capability_before_enqueue() { + for mask in 0..8 { + for (key, modifiers) in [ + ("a", vec!["control"]), + ("p", vec!["control", "shift"]), + ("j", vec!["control"]), + ("p", vec!["control"]), + ("f5", vec![]), + ("slash", vec!["control"]), + ("numpad_1", vec!["option"]), + ] { + let registry = RunnerRegistry::default(); + let mut registration = runner_registration("chords", "instance", Vec::new()); + registration.owner = Some("alice".into()); + registration.capabilities.computer_key_input = mask & 1 != 0; + registration.capabilities.computer_letter_shortcuts = mask & 2 != 0; + registration.capabilities.computer_key_chords = mask & 4 != 0; + registry + .register(current_runner_registration(registration)) + .await + .unwrap(); + let access = auth_context(Some("alice"), false); + let payload = serde_json::json!({"surface_id":"surface_test", "key":key, + "modifiers":modifiers}) + .to_string(); + let result = registry + .enqueue_computer( + "chords".into(), + "computer_key_input", + payload.clone(), + "alice".into(), + Some(&access), + 5, + ) + .await; + let legacy = matches!(key, "j" | "p") && modifiers == ["control"]; + let accepted = mask & 1 != 0 && (mask & 4 != 0 || (legacy && mask & 2 != 0)); + assert_eq!(result.is_ok(), accepted, "{key} {modifiers:?} mask={mask}"); + let request = registry + .poll(RunnerPollRequest { + client_id: "chords".into(), + runner_instance_id: "instance".into(), + }) + .await + .unwrap(); + if accepted { + assert_eq!(request.unwrap().stdin.as_deref(), Some(payload.as_str())); + } else { + assert!(request.is_none()); + assert!(result.unwrap_err().contains(if mask & 1 == 0 { + "computer_key_input" + } else if legacy { + "computer_letter_shortcuts" + } else { + "computer_key_chords" + })); + } + } + } +} + +#[tokio::test] +async fn computer_letter_shortcuts_reject_stale_instances_and_unsupported_chords() { let registry = RunnerRegistry::default(); let mut registration = runner_registration("letters", "new-instance", Vec::new()); registration.owner = Some("alice".into()); @@ -138,17 +209,25 @@ async fn computer_letter_shortcuts_reject_stale_instances_and_invalid_chords() { let payload = serde_json::json!({"surface_id":"surface_test", "key":"p", "modifiers":modifiers}) .to_string(); - assert!(registry + let error = registry .enqueue_computer( "letters".into(), "computer_key_input", payload, "alice".into(), Some(&access), - 5 + 5, ) .await - .is_err()); + .unwrap_err(); + assert!( + error.contains(if modifiers.is_null() { + "invalid computer key input payload" + } else { + "computer_key_chords" + }), + "{error}" + ); } assert!(registry .poll(RunnerPollRequest { diff --git a/crates/webcodex-runner/src/main_tests/registration.rs b/crates/webcodex-runner/src/main_tests/registration.rs index ff17f7d68..27f3e7297 100644 --- a/crates/webcodex-runner/src/main_tests/registration.rs +++ b/crates/webcodex-runner/src/main_tests/registration.rs @@ -281,6 +281,7 @@ fn computer_register_request_announces_platform_capabilities_and_generation() { "computer key input is independently advertised only by native macOS/Windows implementations" ); assert_eq!(caps.computer_letter_shortcuts, cfg!(windows)); + assert_eq!(caps.computer_key_chords, cfg!(windows)); assert_eq!( caps.computer_window_activate, cfg!(any(target_os = "macos", windows)), diff --git a/crates/webcodex-runner/src/webcodex_runner/transport/registration.rs b/crates/webcodex-runner/src/webcodex_runner/transport/registration.rs index 9276c1643..a0d9d6fa7 100644 --- a/crates/webcodex-runner/src/webcodex_runner/transport/registration.rs +++ b/crates/webcodex-runner/src/webcodex_runner/transport/registration.rs @@ -456,6 +456,7 @@ pub(crate) fn runner_register_capabilities(cfg: &RunnerConfig) -> RunnerCapabili cfg!(any(target_os = "macos", windows)), ); capabilities.set(RunnerCapabilityId::ComputerLetterShortcuts, cfg!(windows)); + capabilities.set(RunnerCapabilityId::ComputerKeyChords, cfg!(windows)); // Exact window activation is a separate effect/wire capability. It is // independently advertised by native macOS and Windows implementations. capabilities.set( diff --git a/crates/webcodex-tool-contracts/src/registry/output_schemas/computer.rs b/crates/webcodex-tool-contracts/src/registry/output_schemas/computer.rs index 17007c689..f7d31e9d6 100644 --- a/crates/webcodex-tool-contracts/src/registry/output_schemas/computer.rs +++ b/crates/webcodex-tool-contracts/src/registry/output_schemas/computer.rs @@ -47,9 +47,12 @@ fn target_schema() -> Value { "computer_clipboard_read": {"type": "boolean"}, "computer_clipboard_write": {"type": "boolean"}, "computer_snapshot_region": {"type": "boolean"}, - "computer_accessibility_observe": {"type": "boolean"} + "computer_accessibility_observe": {"type": "boolean"}, + "computer_key_input": {"type": "boolean"}, + "computer_letter_shortcuts": {"type": "boolean"}, + "computer_key_chords": {"type": "boolean"} }, - "required": ["computer_observe", "computer_application_discovery", "computer_application_launch", "computer_display_observe", "computer_pointer_control", "computer_clipboard_read", "computer_clipboard_write", "computer_snapshot_region", "computer_accessibility_observe"] + "required": ["computer_observe", "computer_application_discovery", "computer_application_launch", "computer_display_observe", "computer_pointer_control", "computer_clipboard_read", "computer_clipboard_write", "computer_snapshot_region", "computer_accessibility_observe", "computer_key_input", "computer_letter_shortcuts", "computer_key_chords"] } }, "required": ["client_id", "display_name", "connected", "capabilities"] diff --git a/crates/webcodex-tool-contracts/src/tests/computer_schemas.rs b/crates/webcodex-tool-contracts/src/tests/computer_schemas.rs index f728404df..c036ee80d 100644 --- a/crates/webcodex-tool-contracts/src/tests/computer_schemas.rs +++ b/crates/webcodex-tool-contracts/src/tests/computer_schemas.rs @@ -287,10 +287,7 @@ fn computer_key_schema_discloses_closed_vocabulary_and_modifier_constraints() { let specs = registered_tool_specs(); let spec = spec_named(&specs, "control_computer"); let key = action_properties(&spec.input_schema, "key"); - let keys: Vec<_> = COMPUTER_KEY_INPUT_KEYS - .iter() - .chain(webcodex_core::computer_input_contract::COMPUTER_LETTER_SHORTCUT_KEYS) - .collect(); + let keys: Vec<_> = webcodex_core::computer_input_contract::computer_key_input_keys().collect(); assert_eq!(key["key"]["enum"], json!(keys)); let modifiers = key["modifiers"]["anyOf"] .as_array() @@ -320,7 +317,7 @@ fn computer_key_schema_discloses_closed_vocabulary_and_modifier_constraints() { } } for (name, modifiers) in [ - ("p", json!([])), + ("unknown", json!([])), ("Ctrl+P", json!([])), ("Enter", json!([])), ("tab", json!(["ctrl"])), @@ -360,23 +357,31 @@ fn computer_key_schema_discloses_closed_vocabulary_and_modifier_constraints() { } #[test] -fn computer_key_letter_shortcuts_require_the_exact_control_chord() { +fn computer_key_chords_accept_canonical_keys_and_reject_malformed_modifiers() { let schema = crate::input_schema_for_tool("control_computer"); - for key in ["j", "p"] { - let input = json!({"action":"key", "client_id":"client", "surface_id":"surface_test", "key":key, "modifiers":["control"]}); - test_support::validate_schema_instance(&input, &schema).unwrap(); - ToolCall::from_tool_name("control_computer", input.clone()).unwrap(); + for key in ["a", "j", "p", "0", "f24", "delete", "slash", "numpad_enter"] { + let input = + json!({"action":"key", "client_id":"client", "surface_id":"surface_test", "key":key}); for modifiers in [ None, Some(Value::Null), Some(json!([])), Some(json!(["shift"])), - Some(json!(["option"])), - Some(json!(["command"])), + Some(json!(["control"])), + Some(json!(["control", "shift", "option"])), + ] { + let mut valid = input.clone(); + if let Some(modifiers) = modifiers { + valid["modifiers"] = modifiers; + } + test_support::validate_schema_instance(&valid, &schema).unwrap(); + ToolCall::from_tool_name("control_computer", valid).unwrap(); + } + for modifiers in [ Some(json!(["ctrl"])), - Some(json!(["control", "shift"])), - Some(json!(["control", "option"])), Some(json!(["control", "control"])), + Some(json!(["shift", "control", "option", "command", "shift"])), + Some(json!("control")), ] { let mut invalid = input.clone(); invalid.as_object_mut().unwrap().remove("modifiers"); @@ -401,11 +406,7 @@ fn computer_key_success_receipts_fit_primitive_and_gateway_output_schemas() { ("control_computer", &control.output_schema), ]; let mut failures = Vec::new(); - for key in webcodex_core::computer_input_contract::COMPUTER_KEY_INPUT_KEYS - .iter() - .copied() - .chain(["j", "p"]) - { + for key in webcodex_core::computer_input_contract::computer_key_input_keys() { let modifiers = if matches!(key, "j" | "p") { json!(["control"]) } else { @@ -423,7 +424,7 @@ fn computer_key_success_receipts_fit_primitive_and_gateway_output_schemas() { } } assert!(failures.is_empty(), "{}", failures.join("\n")); - for key in ["a", "Ctrl+P", "P"] { + for key in ["unknown", "Ctrl+P", "P"] { let receipt = json!({"success":true, "output":{"platform":"windows", "surface_id":"surface_test", "key":key, "modifiers":["control"], "success":true}}); for (name, schema) in output_schemas { diff --git a/crates/webcodex-tool-contracts/src/tool_call/input_types.rs b/crates/webcodex-tool-contracts/src/tool_call/input_types.rs index 5d870c5c6..98ebeb557 100644 --- a/crates/webcodex-tool-contracts/src/tool_call/input_types.rs +++ b/crates/webcodex-tool-contracts/src/tool_call/input_types.rs @@ -1400,18 +1400,17 @@ pub enum ComputerControlToolCall { #[schemars(length(min = 1, max = 128))] element_id: String, }, - #[schemars(extend("allOf" = webcodex_core::computer_input_contract::computer_key_chord_constraints()))] Key { #[schemars(length(min = 1, max = 128))] client_id: String, #[schemars(length(min = 1, max = 128))] surface_id: String, - /// Exact lowercase navigation key, or Windows-only j/p with exactly modifiers=[control]. + /// Exact canonical key name; Windows supports general single-key chords. /// Shortcut strings such as Ctrl+P are unsupported. The exact surface must already be foreground/focused. #[schemars(schema_with = "webcodex_core::computer_input_contract::computer_key_schema")] key: String, /// Unique canonical modifiers; omission/null means none. Windows maps option to Alt, rejects - /// command, option+Tab, option+Escape, and any control+Escape chord. macOS uses the named + /// command, option+Tab, option+Escape, control+Escape and control+option+Delete. macOS uses the named /// Shift/Control/Option/Command flags. Held/interfering keys can fail native preflight. #[schemars(schema_with = "webcodex_core::computer_input_contract::computer_key_modifiers_schema")] #[serde(default)] diff --git a/crates/webcodex-tool-contracts/src/tool_definition/computer.rs b/crates/webcodex-tool-contracts/src/tool_definition/computer.rs index a32021c2c..e06d3bdbb 100644 --- a/crates/webcodex-tool-contracts/src/tool_definition/computer.rs +++ b/crates/webcodex-tool-contracts/src/tool_definition/computer.rs @@ -37,7 +37,7 @@ pub(super) const DEFINITIONS: &[ToolDefinition] = &[ super::ToolSemanticContract { effect: super::ToolEffect::Execute, risk: ComputerControlRisk, approval: super::ToolApprovalPolicy::Standard, idempotency: super::ToolIdempotency::NonIdempotent }, None, false, NoPath, true, false, super::ToolSessionEvidencePolicy::NONE, ), - "Effectful Computer gateway: launch_application, activate_window, press, focus, scroll_to_element, key, input_text, pointer_move, pointer_click, write_clipboard. If unsure, query observe_computer(action=element_state) on exact client_id/surface_id/element_id: can_press/can_focus/can_input_text guide actions; avoid unchanged retries on false; true grants no authority. Windows press needs InvokePattern; focus/input need foreground AXTextField; input needs exact focus and writable empty value. key accepts navigation enums; Windows Ctrl+J/Ctrl+P use key=j/p,modifiers=[control] exactly and require computer_letter_shortcuts plus computer_key_input. No other letter chords or shortcut strings. Virtual keys are not text or guaranteed app commands. Shift+Tab: key=tab,modifiers=[shift]. Windows option=Alt, command unsupported; option+Tab, option+Escape, control+Escape rejected. macOS uses named modifiers. Surface must be foreground/focused, with unprotected focus and native preflight; release held keys before re-observing. press/input_text may use observe_after={action:accessibility_tree,max_depth?,max_nodes?}: requires computer:read and accessibility capability before acting, pins Runner/window, returns fresh bounded output.post_observation={success,output,error?}. Tree refresh invalidates old IDs; not UI-settled. Read failure preserves action success; uncertain action skips read. Scopes, permissions, capabilities, identities and snapshot guards apply. No implicit focus/activation, script fallback or blind effect replay.", + "Effectful Computer gateway: launch_application, activate_window, press, focus, scroll_to_element, key, input_text, pointer_move, pointer_click, write_clipboard. If unsure, query observe_computer(action=element_state) on exact client_id/surface_id/element_id: can_press/can_focus/can_input_text advise actions; false avoids unchanged retries; true grants no authority. Windows press needs InvokePattern; focus/input need foreground AXTextField; input needs exact focus and writable empty value. key uses schema names; Windows keys need computer_key_input; extra keys/chords need computer_key_chords or legacy computer_letter_shortcuts for exact Ctrl+J/Ctrl+P. macOS navigation only. Key events are not text or guaranteed app commands. Shift+Tab: key=tab,modifiers=[shift]. Windows option=Alt, command unsupported; option+Tab, option+Escape, control+Escape, control+option+Delete rejected. Surface must be foreground/focused; protected focus rejected; release held keys and re-observe. press/input_text accept observe_after={action:accessibility_tree,max_depth?,max_nodes?}: needs computer:read and accessibility capability preflight, pins Runner/window, returns bounded output.post_observation={success,output,error?}. Tree refresh invalidates IDs; not UI-settled. Read failure preserves action success; uncertain action skips read. Scopes, permissions, capabilities, identities and snapshot guards apply. No implicit focus/activation, scripts or blind effect replay.", ), PERMISSION_RISK_WRITE, ), diff --git a/docs/COMPUTER_USE.md b/docs/COMPUTER_USE.md index aa25c54fd..ca94722a2 100644 --- a/docs/COMPUTER_USE.md +++ b/docs/COMPUTER_USE.md @@ -214,15 +214,24 @@ On macOS the Runner keeps the existing exact surface/element correlation and pro No lower-level wheel fallback is added in this slice. If a concrete product need emerges later, it must remain separately bounded and tied to an exactly revalidated active/focused surface rather than widening this semantic contract. -### CU-10 — closed key input +### CU-10 — bounded key chords -The original navigation vocabulary uses the exact lowercase keys: `enter`, `escape`, `tab`, `arrow_up`, `arrow_down`, `arrow_left`, `arrow_right`, `page_up`, `page_down`, `home`, `end`. The optional `modifiers` array accepts only unique `shift`, `control`, `option`, and `command` values (at most four); omission or an empty array means no modifiers. The schema and Server/Computer validators share `webcodex_core::computer_input_contract`; the Server normalizes modifier order to that canonical order. These are vocabulary limits, not a promise that every chord is supported on every platform. +Each call sends one main key with an optional set of unique modifiers. The schema and Server/Computer validators share `webcodex_core::computer_input_contract`; the Server normalizes modifiers to `shift`, `control`, `option`, `command` order. Omission, null, or an empty array means no modifiers. Windows supports Shift, Control and Alt (`option`); `command` has no Windows mapping. macOS retains the original eleven navigation keys with its existing modifier mappings. The shared vocabulary has 102 exact lowercase names: -For example, Shift+Tab uses `{"action":"key","client_id":"","surface_id":"","key":"tab","modifiers":["shift"]}`. Windows additionally supports only `Ctrl+J` and `Ctrl+P`: use `key: "j"` or `key: "p"` with exactly `modifiers: ["control"]`. Standalone letters, additional modifiers, shortcut strings such as `"Ctrl+P"`, and aliases such as `ctrl`/`alt` are rejected. Schema, Server, admission and native validation enforce the complete chord. On Windows, `option` means Alt, `command` is unsupported, and any chord containing Alt+Tab, Alt+Escape, or Ctrl+Escape is rejected before native input. Windows also rejects interfering held keys. The exact window must already be foreground/focused; key input never activates or focuses it implicitly. Capability and native preflight checks remain authoritative, and an uncertain effect still requires re-observation before retry. +| Group | Key names | +| --- | --- | +| Navigation | `enter`, `escape`, `tab`, `arrow_up`, `arrow_down`, `arrow_left`, `arrow_right`, `page_up`, `page_down`, `home`, `end` | +| Letters and digits | `a`–`z`, `0`–`9` | +| Function | `f1`–`f24` | +| Editing | `space`, `backspace`, `delete`, `insert` | +| Punctuation | `semicolon`, `equal`, `comma`, `minus`, `period`, `slash`, `backquote`, `left_bracket`, `backslash`, `right_bracket`, `quote` | +| Keypad | `numpad_0`–`numpad_9`, `numpad_add`, `numpad_subtract`, `numpad_multiply`, `numpad_divide`, `numpad_decimal`, `numpad_enter` | -Expose key input as `control_computer(action=key, client_id, surface_id, key, modifiers?)` while retaining the additive `computer_key_input` Runner capability and wire kind. An older Runner that only advertises the semantic `control_computer` capability still fails closed. The navigation vocabulary remains Enter, Escape, Tab, arrows, PageUp/PageDown, Home/End with the bounded `shift`, `control`, `option`, and `command` modifier set. The Windows letter extension additionally requires `computer_letter_shortcuts`, advertised only by supporting Windows Runners; older Runners and other platforms are rejected before dispatch. Letter requests pin the accepted Runner instance and recheck access and both capabilities at queue admission and dequeue. Ordinary text continues through `control_computer(action=input_text)`; callers cannot supply characters, virtual keycodes, repeat counts, held-key state, or modifier key-down/up sequences. +For example, Ctrl+Shift+P uses `{"action":"key","client_id":"","surface_id":"","key":"p","modifiers":["control","shift"]}`. Bare keys and other supported modifier combinations use the same shape. Shortcut strings such as `"Ctrl+P"`, aliases such as `ctrl`/`alt`, duplicate modifiers, and caller-defined keycodes are rejected. Windows rejects any chord containing Alt+Tab, Alt+Escape, Ctrl+Escape, or Ctrl+Alt+Delete before native input, as well as interfering held keys. The exact window must already be foreground/focused; key input never activates or focuses it implicitly. An uncertain effect requires re-observation before considering a retry. -Windows letter shortcuts use the existing bounded `SendInput` path with virtual keys `VK_J`/`VK_P` and Control. The key names denote those virtual keys, not Unicode text or a physical scan-code position. The complete down/up sequence is prepared before injection; existing foreground/focus, protected-content, held-key and partial-delivery checks remain. Windows uses the shared interactive input stream: application bindings, keyboard layout/input-method state, global hotkeys and hooks can affect the resulting behavior. Foreground preflight is not desktop isolation or proof that an application command completed. Observe the resulting UI; a newly opened dialog needs a fresh observed identity. A failed or missing response must not cause automatic replay or a guessed key-release sequence. +Key input retains `control_computer(action=key, client_id, surface_id, key, modifiers?)`, the `computer_key_input` wire kind, and the independent `computer_key_input` base capability. Extra keys require the additive Windows-only `computer_key_chords` capability. The earlier F02 Server/Runner wire consumer remains supported through `computer_letter_shortcuts` for exactly Ctrl+J and Ctrl+P; this bit does not grant other letter/modifier combinations. A general-chord Runner needs no legacy letter bit. `observe_computer(action=targets)` reports all three actual capability bits so independently upgraded Servers and Runners can be distinguished. All key requests pin the accepted Runner instance and recheck access and the required capabilities at queue admission and dequeue. An older Runner advertising only semantic `computer_control` is not key-input capable. Ordinary text continues through `control_computer(action=input_text)`; callers cannot supply Unicode characters, virtual keycodes, repeat counts, held-key state, or modifier key-down/up sequences. + +Windows chords use the existing bounded `SendInput` path with logical virtual keys. Punctuation names map to the corresponding `VK_OEM_*` keys; they do not promise a particular character under another keyboard layout. Keypad names preserve their native virtual-key and extended-key distinctions; NumLock and input-method state can affect application behavior and are not changed by this tool. The complete down/up sequence is prepared before injection, with at most eight events on Windows; foreground/focus, protected-content, held-key and partial-delivery checks remain. Application bindings, global hotkeys and hooks also affect behavior. A successful receipt proves the complete sequence entered the shared input stream, not that an application command completed. Observe the resulting UI; a newly opened dialog needs a fresh observed identity. A failed or missing response must not cause automatic replay or a guessed key-release sequence. On macOS the Runner revalidates the exact surface through the existing native window identity path, then requires its application to already be frontmost and `AXFocusedWindow` to equal that exact window. It never activates or focuses the surface implicitly. When an `AXFocusedUIElement` is available, explicitly protected or secure content fails closed. Accessibility and event-posting permission checks are preflight-only and do not request permission UI. Both key-down and key-up events are constructed and flagged before the first effect, then posted only to the exact surface PID. Quartz exposes no atomic/batch post for the pair, so the two posts are adjacent with no fallible work between them; if the Runner process terminates in that narrow interval, the result is a partial `outcome_unknown`, not a supported held-key state. @@ -270,11 +279,11 @@ Windows uses `control_computer(action=scroll_to_element, surface_id, element_id) Missing `ScrollItemPattern` and other pre-effect validation failures are deterministic. Once `ScrollIntoView` has been invoked, any HRESULT failure or deadline loss is `outcome_unknown`; callers must make a fresh semantic observation before deciding whether another scroll is safe. The result remains closed metadata containing only platform, surface/element identity, and success. -### Windows UIA parity W6 — closed key input +### Windows UIA parity W6 — bounded key chords Windows uses `control_computer(action=key, surface_id, key, modifiers?)` while retaining the independent `computer_key_input` Runner capability. Immediately before input the Runner revalidates the exact xcap surface, HWND/PID UIA root, requires `GetForegroundWindow()` to equal that HWND, obtains the current focused UIA element, and walks a bounded Control View ancestry chain until it proves that focus belongs to the exact window root. Password/protected focused content or ancestry fails closed. The Runner performs this exact foreground/root/focus preflight again immediately before native input and never activates or focuses a window implicitly. -The existing closed key vocabulary is mapped to native Windows virtual keys internally. `shift` maps to Shift, `control` to Control, and model-facing `option` to Alt. `command` deliberately has no Windows mapping and fails before any native input. Windows also rejects `option+Tab`, `option+Escape`, and any `control+Escape` chord before native input because those combinations invoke OS-level switching or shell UI outside the exact surface. The full bounded `INPUT` sequence is constructed before the first effect: modifier downs, one key down/up pair, then modifier ups in reverse order. Navigation keys that require it use the internally selected extended-key flag. The entire sequence is sent with one `SendInput` call and the returned inserted-event count must exactly equal the prepared sequence length. Immediately before that call, the Runner also fails closed if Shift, Control, Alt, either Windows key, or the requested target key is already physically down, because existing keyboard state can otherwise change the effective chord. +The shared key vocabulary maps to native Windows virtual keys internally. `shift` maps to Shift, `control` to Control, and model-facing `option` to Alt. `command` deliberately has no Windows mapping and fails before any native input. Windows also rejects `option+Tab`, `option+Escape`, `control+Escape`, and `control+option+Delete` chords, including supersets, before native input. The full bounded `INPUT` sequence is constructed before the first effect: modifier downs, one key down/up pair, then modifier ups in reverse order. Navigation, Delete/Insert, keypad Divide and keypad Enter use their required extended-key flags. The entire sequence is sent with one `SendInput` call and the returned inserted-event count must exactly equal the prepared sequence length. Immediately before that call, the Runner also fails closed if Shift, Control, Alt, either Windows key, or the requested target key is already physically down, because existing keyboard state can otherwise change the effective chord. Windows `SendInput` uses the shared interactive-desktop input stream; the exact surface/focus checks and held-key guard bound the action but do not make it a concurrent-user isolation primitive. Do not assume Windows key input can run independently while a person is simultaneously changing focus or keyboard state on the same desktop. Before `SendInput`, deterministic foreground/focus/mapping/held-key failures are pre-effect. A zero inserted-event return is also deterministic because no keyboard event entered the input stream. Partial insertion, deadline loss after insertion, interruption, lost response, or inconsistent success metadata is `outcome_unknown`; callers must re-observe before considering a retry. There is no PostMessage/SendMessage, `keybd_event`, caller virtual-key/scan-code control, clipboard/paste, script/shell, coordinate/pointer fallback, implicit activation, or cross-call held-key state. Successful output remains closed metadata containing only platform, surface identity, key, modifiers, and success. diff --git a/src/mcp_tests/computer_contracts.rs b/src/mcp_tests/computer_contracts.rs index bb8c1a6e4..7318cf5cb 100644 --- a/src/mcp_tests/computer_contracts.rs +++ b/src/mcp_tests/computer_contracts.rs @@ -18,7 +18,7 @@ fn assert_computer_key_manifest_contract(schema: &Value) { assert!(crate::tool_runtime::ToolCall::from_tool_name("control_computer", input).is_ok()); } for (key, modifiers) in [ - ("p", json!([])), + ("unknown", json!([])), ("Ctrl+P", json!([])), ("Enter", json!([])), ("tab", json!(["ctrl"])), diff --git a/src/tool_runtime/computer_tools/observe.rs b/src/tool_runtime/computer_tools/observe.rs index 2aef44001..64bf49a4b 100644 --- a/src/tool_runtime/computer_tools/observe.rs +++ b/src/tool_runtime/computer_tools/observe.rs @@ -289,6 +289,9 @@ impl ToolRuntime { let computer_snapshot_region = client.supports(RunnerFeature::ComputerSnapshotRegion); let computer_accessibility_observe = client.supports(RunnerFeature::ComputerAccessibilityObserve); + let computer_key_input = client.supports(RunnerFeature::ComputerKeyInput); + let computer_letter_shortcuts = client.supports(RunnerFeature::ComputerLetterShortcuts); + let computer_key_chords = client.supports(RunnerFeature::ComputerKeyChords); let view = client.view; if !computer_observe && !computer_accessibility_observe @@ -298,6 +301,7 @@ impl ToolRuntime { && !computer_pointer_control && !computer_clipboard_read && !computer_clipboard_write + && !computer_key_input { continue; } @@ -319,6 +323,9 @@ impl ToolRuntime { "computer_clipboard_write": computer_clipboard_write, "computer_snapshot_region": computer_snapshot_region, "computer_accessibility_observe": computer_accessibility_observe, + "computer_key_input": computer_key_input, + "computer_letter_shortcuts": computer_letter_shortcuts, + "computer_key_chords": computer_key_chords, }, })); } diff --git a/src/tool_runtime/computer_tools/request.rs b/src/tool_runtime/computer_tools/request.rs index c4af3dca5..a8d6e43f1 100644 --- a/src/tool_runtime/computer_tools/request.rs +++ b/src/tool_runtime/computer_tools/request.rs @@ -69,11 +69,19 @@ impl ToolRuntime { accessibility_bounds: Option<(usize, usize)>, expected_instance: Option<&str>, ) -> ToolResult { - let is_letter_shortcut = kind == "computer_key_input" - && payload - .get("key") - .and_then(Value::as_str) - .is_some_and(webcodex_core::computer_input_contract::is_computer_letter_shortcut); + let key_input = if kind == "computer_key_input" { + let Ok(input) = serde_json::from_value::< + webcodex_core::computer_input_contract::ComputerKeyInputPayload, + >(payload.clone()) else { + return computer_error("invalid_request", "invalid computer key input payload"); + }; + if let Err(error) = input.validate() { + return computer_error("invalid_request", error); + } + Some(input) + } else { + None + }; let expected_root_element_id = payload .get("root_element_id") .and_then(Value::as_str) @@ -158,14 +166,7 @@ impl ToolRuntime { "computer_element_state" => &[RunnerFeature::ComputerElementState], "computer_control" => &[RunnerFeature::ComputerControl], "computer_scroll_to_element" => &[RunnerFeature::ComputerScrollToElement], - "computer_key_input" => { - webcodex_core::computer_input_contract::computer_key_input_capabilities( - payload - .get("key") - .and_then(Value::as_str) - .unwrap_or_default(), - ) - } + "computer_key_input" => &[RunnerFeature::ComputerKeyInput], "computer_activate_window" => &[RunnerFeature::ComputerWindowActivate], "computer_input_text" => &[RunnerFeature::ComputerTextInput], _ => return computer_error("invalid_request", "unsupported computer request kind"), @@ -204,17 +205,27 @@ impl ToolRuntime { ); } Err(error) - if (expected_instance.is_some() || is_letter_shortcut) + if (expected_instance.is_some() || key_input.is_some()) && computer_request_is_effect(kind) => { return computer_effect_not_started(&error); } Err(error) => return computer_error("client_access_denied", &error), }; - // Pin new letter chords to the registration whose capability was observed. + // Pin key input to the registration whose capability was observed. // The same fence is checked under the queue lock and again at dequeue. - let expected_instance = expected_instance - .or_else(|| is_letter_shortcut.then_some(client.view.runner_instance_id.as_str())); + let expected_instance = expected_instance.or_else(|| { + key_input + .is_some() + .then_some(client.view.runner_instance_id.as_str()) + }); + let required_features = key_input.as_ref().map_or(required_features, |input| { + webcodex_core::computer_input_contract::computer_key_input_capabilities( + &input.key, + &input.modifiers, + client.supports(RunnerFeature::ComputerKeyChords), + ) + }); for required_feature in required_features { if client.supports(*required_feature) { continue; diff --git a/src/tool_runtime/computer_tools_tests.rs b/src/tool_runtime/computer_tools_tests.rs index 1a4384c8d..d682b9203 100644 --- a/src/tool_runtime/computer_tools_tests.rs +++ b/src/tool_runtime/computer_tools_tests.rs @@ -1,5 +1,128 @@ use super::responses::{validate_computer_response, ComputerResponseContext}; use super::*; +use crate::tool_runtime::RuntimeInfo; + +#[tokio::test] +async fn computer_targets_expose_authoritative_key_capabilities() { + use crate::runner_protocol::RunnerCapabilities; + use std::sync::Arc; + + let registry = Arc::new(crate::runner_http::RunnerRegistry::default()); + for (id, base, letters, chords) in [ + ("navigation", true, false, false), + ("legacy", true, true, false), + ("general", true, false, true), + ("unusable", false, false, true), + ] { + registry + .register(crate::test_support::runner_registration( + id, + "instance", + RunnerCapabilities { + computer_key_input: base, + computer_letter_shortcuts: letters, + computer_key_chords: chords, + ..Default::default() + }, + )) + .await + .unwrap(); + } + let runtime = ToolRuntime::new(registry, Arc::new(RuntimeInfo::default())); + let mut auth = AuthContext::new(crate::auth::AuthKind::Bootstrap); + auth.is_bootstrap = true; + let result = runtime.computer_list_targets(Some(&auth)).await; + assert!(result.success, "{result:?}"); + assert_eq!(result.output["count"], 3); + let schema = webcodex_tool_contracts::output_schema_for_tool("observe_computer"); + webcodex_tool_contracts::test_support::validate_schema_instance( + &serde_json::to_value(&result).unwrap(), + &schema, + ) + .unwrap(); + let targets = result.output["targets"].as_array().unwrap(); + for (id, letters, chords) in [ + ("navigation", false, false), + ("legacy", true, false), + ("general", false, true), + ] { + let target = targets + .iter() + .find(|target| target["client_id"] == id) + .unwrap(); + assert_eq!(target["capabilities"]["computer_key_input"], true); + assert_eq!(target["capabilities"]["computer_letter_shortcuts"], letters); + assert_eq!(target["capabilities"]["computer_key_chords"], chords); + } +} + +#[tokio::test] +async fn computer_general_key_admission_rejections_are_not_started() { + use crate::runner_protocol::{RunnerCapabilities, RunnerPollRequest}; + use std::sync::Arc; + + for replaced in [false, true] { + for (key, modifiers) in [ + ("a", vec!["control"]), + ("p", vec!["shift", "control"]), + ("tab", vec!["shift"]), + ] { + // Navigation is used only for the instance fence; it needs no new bit. + if !replaced && key == "tab" { + continue; + } + let registry = Arc::new(crate::runner_http::RunnerRegistry::default()); + registry + .register(crate::test_support::runner_registration( + "computer", + "current", + RunnerCapabilities { + computer_key_input: true, + computer_letter_shortcuts: true, + computer_key_chords: replaced, + ..Default::default() + }, + )) + .await + .unwrap(); + let runtime = ToolRuntime::new(registry.clone(), Arc::new(RuntimeInfo::default())); + let mut auth = AuthContext::new(crate::auth::AuthKind::Bootstrap); + auth.is_bootstrap = true; + let result = runtime + .dispatch_computer_request_for_instance( + "computer", + "computer_key_input", + json!({"surface_id":"surface_test", "key":key, + "modifiers":modifiers}), + Some(&auth), + None, + Some("surface_test"), + None, + Some(if replaced { "previous" } else { "current" }), + ) + .await; + assert!(!result.success, "{key}: {result:?}"); + assert_eq!(result.output["execution_state"], "not_started"); + assert_eq!(result.output["state_changed"], false); + assert!( + result.error.as_deref().unwrap().contains(if replaced { + "runner_replaced:" + } else { + "computer_key_chords" + }), + "{result:?}" + ); + assert!(registry + .poll(RunnerPollRequest { + client_id: "computer".into(), + runner_instance_id: "current".into() + }) + .await + .unwrap() + .is_none()); + } + } +} #[tokio::test] async fn computer_post_observation_admission_rejection_preserves_not_started() { @@ -1483,7 +1606,7 @@ fn computer_key_input_normalizes_closed_vocabulary_and_modifiers() { .unwrap(), vec!["shift".to_string(), "command".to_string()] ); - assert!(normalize_computer_key_input("a", None).is_err()); + assert!(normalize_computer_key_input("unknown", None).is_err()); assert!(normalize_computer_key_input( "enter", Some(vec!["shift".to_string(), "shift".to_string()]), @@ -1494,8 +1617,8 @@ fn computer_key_input_normalizes_closed_vocabulary_and_modifiers() { } #[test] -fn computer_key_input_letter_chords_are_not_text_or_arbitrary_shortcuts() { - for key in ["j", "p"] { +fn computer_key_input_general_chords_use_canonical_keys_and_modifiers() { + for key in webcodex_core::computer_input_contract::computer_key_input_keys() { assert_eq!( normalize_computer_key_input(key, Some(vec!["control".into()])).unwrap(), vec!["control"] @@ -1507,10 +1630,10 @@ fn computer_key_input_letter_chords_are_not_text_or_arbitrary_shortcuts() { Some(vec!["control".into(), "option".into()]), Some(vec!["control".into(), "shift".into()]), ] { - assert!(normalize_computer_key_input(key, modifiers).is_err()); + assert!(normalize_computer_key_input(key, modifiers).is_ok()); } } - for key in ["a", "P", "Ctrl+P", "0x50", "j p"] { + for key in ["unknown", "P", "Ctrl+P", "0x50", "j p"] { assert!(normalize_computer_key_input(key, Some(vec!["control".into()])).is_err()); } } @@ -1559,20 +1682,35 @@ fn computer_key_input_validator_is_exact_and_post_dispatch_mismatch_is_unknown() #[test] fn computer_key_input_validator_accepts_closed_windows_metadata() { - let expected_modifiers = json!(["shift"]); - let valid = validate_computer_key_input( - json!({ - "platform": "windows", - "surface_id": "surface_test", - "key": "tab", - "modifiers": ["shift"], - "success": true - }), - "surface_test", - "tab", - &expected_modifiers, - ); - assert!(valid.success, "{:?}", valid.output); + for key in webcodex_core::computer_input_contract::computer_key_input_keys() { + let expected_modifiers = json!(["shift"]); + let valid = validate_computer_key_input( + json!({ + "platform": "windows", + "surface_id": "surface_test", + "key": key, + "modifiers": ["shift"], + "success": true + }), + "surface_test", + key, + &expected_modifiers, + ); + assert!(valid.success, "{:?}", valid.output); + let schema = webcodex_tool_contracts::output_schema_for_tool("control_computer"); + webcodex_tool_contracts::test_support::validate_schema_instance( + &serde_json::to_value(&valid).unwrap(), + &schema, + ) + .unwrap(); + let mut receipt = valid.output; + receipt["key"] = json!(if key == "a" { "p" } else { "a" }); + let invalid = + validate_computer_key_input(receipt, "surface_test", key, &expected_modifiers); + assert!(!invalid.success, "{key}"); + let unknown = computer_effect_validated_result(invalid, "observe before retrying"); + assert_eq!(unknown.output["execution_state"], "outcome_unknown"); + } } #[test]