diff --git a/skills/autonomous-ai-agents/computer-use/SKILL.md b/skills/autonomous-ai-agents/computer-use/SKILL.md index 1bd441a211..8e6b001650 100644 --- a/skills/autonomous-ai-agents/computer-use/SKILL.md +++ b/skills/autonomous-ai-agents/computer-use/SKILL.md @@ -140,6 +140,17 @@ Walk it in order: (it's a visible focus change) and is only appropriate when the user isn't actively working. Classic cases: Electron/Chromium consent dialogs (e.g. tldraw offline's "Run Script"), DirectInput games, raw-input canvases. +6. **Keystrokes verified-lost on a KDE/Qt editor → use the app's own I/O.** + Some Qt text components (KTextEditor: Kate, KWrite, KDevelop) discard + SYNTHETIC X keystrokes entirely — foreground `type` reports ok + ("Typed N characters into the focused widget", `effect:"unverifiable"`) + but a fresh AX capture shows the text never arrived, and raw XTest fails + identically (proven live, Aug 2026 — it is the toolkit, not the driver; + the same foreground route works on kcalc/Chrome). After ONE such + verified-lost round trip, stop retrying input rungs: write the file with + terminal/file tools and let the editor reload it, or drive the app's + DBus/CLI interface. Never loop the ladder against a surface that + verifiably swallows synthetic input. ``` computer_use(action="click", element=7) diff --git a/tests/tools/test_computer_use_input_target_guard.py b/tests/tools/test_computer_use_input_target_guard.py new file mode 100644 index 0000000000..3d80c922e0 --- /dev/null +++ b/tests/tools/test_computer_use_input_target_guard.py @@ -0,0 +1,113 @@ +"""Input actions must not silently deliver to a different app than requested. + +Live QA (Aug 2026, KDE desktop): with kcalc as the sticky target, +``type(text="777", app="kate")`` reported ok:true and typed 777 into +KCALC — app= was silently dropped on every input action. The guard +refuses provable mismatches with a one-call fix instruction. Also covers +the unknown-action suggestion hint (model emitted "hotkey" for "key"). +""" +import json +from types import SimpleNamespace + +from tools.computer_use.tool import ( + _INPUT_ACTIONS, + _dispatch, + _input_target_mismatch, +) + + +def _backend(last_app): + return SimpleNamespace(_last_app=last_app) + + +# ── mismatch predicate ────────────────────────────────────────────────── + + +def test_clear_mismatch_returns_current_app(): + assert _input_target_mismatch(_backend("kcalc"), "kate") == "kcalc" + + +def test_same_app_no_mismatch(): + assert _input_target_mismatch(_backend("kate"), "kate") is None + + +def test_substring_variants_no_mismatch(): + # list_windows names are localized/variant; never refuse fuzzy matches. + assert _input_target_mismatch(_backend("Google-chrome"), "chrome") is None + assert _input_target_mismatch(_backend("chrome"), "Google-chrome") is None + + +def test_unknown_current_target_fails_open(): + assert _input_target_mismatch(_backend(None), "kate") is None + assert _input_target_mismatch(_backend(""), "kate") is None + + +# ── dispatch guard ────────────────────────────────────────────────────── + + +def _dispatch_result(backend, action, args): + return json.loads(_dispatch(backend, action, args)) + + +def test_type_with_mismatched_app_refused(): + backend = _backend("kcalc") + out = _dispatch_result(backend, "type", {"text": "777", "app": "kate"}) + assert out["ok"] is False + assert out["code"] == "input_target_mismatch" + assert "kcalc" in out["error"] and "kate" in out["error"] + assert "capture(app='kate')" in out["error"] + + +def test_click_with_mismatched_app_refused(): + out = _dispatch_result(_backend("kcalc"), "click", {"element": 3, "app": "kate"}) + assert out["code"] == "input_target_mismatch" + + +def _fake_action_result(action="type_text"): + from tools.computer_use.backend import ActionResult + + return ActionResult(ok=True, action=action, message="done") + + +def test_type_with_matching_app_reaches_backend(): + backend = _backend("kate") + calls = {} + + def _type_text(text, **kw): + calls["text"] = text + return _fake_action_result() + + backend.type_text = _type_text + out = _dispatch_result(backend, "type", {"text": "hi", "app": "kate"}) + assert calls["text"] == "hi" + assert out.get("ok") is True + + +def test_type_without_app_unchanged(): + """Legacy flows that never pass app= keep working (fail open).""" + backend = _backend("kcalc") + backend.type_text = lambda text, **kw: _fake_action_result() + out = _dispatch_result(backend, "type", {"text": "9"}) + assert out.get("ok") is True + + +def test_input_actions_set_matches_dispatch_branches(): + # Guard list must cover exactly the sticky-target input actions. + assert _INPUT_ACTIONS == { + "click", "double_click", "right_click", "middle_click", + "drag", "scroll", "type", "key", "set_value", + } + + +# ── unknown-action suggestions ────────────────────────────────────────── + + +def test_unknown_action_hotkey_suggests_key(): + out = _dispatch_result(_backend(None), "hotkey", {}) + assert "did you mean 'key'" in out["error"] + + +def test_unknown_action_without_suggestion_unchanged(): + out = _dispatch_result(_backend(None), "frobnicate", {}) + assert out["error"] == "unknown action 'frobnicate'" + assert "did you mean" not in out["error"] diff --git a/tools/computer_use/tool.py b/tools/computer_use/tool.py index 30490633be..a77de3cd26 100644 --- a/tools/computer_use/tool.py +++ b/tools/computer_use/tool.py @@ -124,6 +124,34 @@ def _canon_key_combo(keys: str) -> frozenset: return frozenset(parts) +# Native input actions that deliver to the backend's sticky target. `app=` +# on these calls is NOT a targeting parameter — see the mismatch guard in +# _dispatch. Kept in sync with the dispatch branches below. +_INPUT_ACTIONS = frozenset({ + "click", "double_click", "right_click", "middle_click", + "drag", "scroll", "type", "key", "set_value", +}) + + +def _input_target_mismatch(backend, requested_app: str) -> Optional[str]: + """Current sticky-target app when it clearly differs from *requested_app*. + + Returns the CURRENT target's app name only for a provable mismatch: + both names known and neither a substring of the other (list_windows + app names are localized/variant — 'Google-chrome' vs 'chrome'). An + unknown current target returns None (fail open: legacy flows that + never pass app= on input keep working; wrong-window delivery there is + caught by the verify ladder instead). + """ + current = (getattr(backend, "_last_app", None) or "").strip().lower() + wanted = requested_app.strip().lower() + if not current or not wanted: + return None + if wanted in current or current in wanted: + return None + return getattr(backend, "_last_app", None) + + # Dangerous text patterns for the `type` action. Same list as #4562. _BLOCKED_TYPE_PATTERNS = [ re.compile(r"curl\s+[^|]*\|\s*bash", re.IGNORECASE), @@ -724,6 +752,31 @@ def _dispatch(backend: ComputerUseBackend, action: str, args: Dict[str, Any]) -> delivery_mode = args.get("delivery_mode") bring_to_front = bool(args.get("bring_to_front")) + # ── app= mismatch guard for input actions ────────────────────────── + # Input goes to the backend's sticky target (set by the last capture/ + # focus_app). Models routinely pass app= on the input call itself — + # live QA (Aug 2026) proved `type(text=..., app="kate")` typed into + # kcalc while reporting ok:true, because the argument was silently + # dropped. Refuse the clear mismatch instead of delivering input to + # the wrong window; the fix instruction keeps the flow one call long. + if action in _INPUT_ACTIONS: + requested_app = args.get("app") + if isinstance(requested_app, str) and requested_app.strip(): + mismatch = _input_target_mismatch(backend, requested_app) + if mismatch is not None: + return json.dumps({ + "ok": False, + "action": action, + "code": "input_target_mismatch", + "error": ( + f"{action} would go to the current target " + f"{mismatch!r}, not {requested_app.strip()!r} — input " + "actions always hit the sticky target from the last " + f"capture/focus_app. Call capture(app={requested_app.strip()!r}) " + "or focus_app first, then retry." + ), + }) + if action in {"click", "double_click", "right_click", "middle_click"}: button = args.get("button") click_count = 1 @@ -794,6 +847,24 @@ def _dispatch(backend: ComputerUseBackend, action: str, args: Dict[str, Any]) -> res = backend.set_value(value=str(value), element=args.get("element")) return _maybe_follow_capture(backend, res, capture_after) + # Do NOT alias unknown actions (we never repair bad model output), but + # name the nearest real action: live QA showed a model emitting + # "hotkey"/"press_key" and getting zero guidance from the bare error. + _suggestions = { + "hotkey": "key", "press_key": "key", "keypress": "key", + "key_combo": "key", "shortcut": "key", + "type_text": "type", "input_text": "type", + "screenshot": "capture", "get_window_state": "capture", + "left_click": "click", "mouse_click": "click", + } + hint = _suggestions.get(str(action)) + if hint: + return json.dumps({ + "error": ( + f"unknown action {action!r} — did you mean {hint!r}? " + "See the action enum in the tool schema." + ) + }) return json.dumps({"error": f"unknown action {action!r}"})