diff --git a/tests/tools/test_computer_use_zero_bounds.py b/tests/tools/test_computer_use_zero_bounds.py new file mode 100644 index 0000000000..9867b4b58d --- /dev/null +++ b/tests/tools/test_computer_use_zero_bounds.py @@ -0,0 +1,54 @@ +"""Zero-rect AX bounds must read as 'unknown', never as a clickable position. + +Live QA (Aug 2026): KDE/Qt apps report [0,0,0,0] for elements that are +perfectly clickable by index (all of kcalc's radio buttons). A +plausible-looking zero rect invites coordinate=[0,0] derivation. +""" +from tools.computer_use.backend import UIElement +from tools.computer_use.tool import ( + _bounds_unknown, + _element_to_dict, + _format_elements, +) + + +def _el(bounds, index=2, role="radio button", label="Deg"): + return UIElement(index=index, role=role, label=label, bounds=tuple(bounds), app="") + + +def test_zero_rect_is_unknown(): + assert _bounds_unknown((0, 0, 0, 0)) is True + + +def test_real_rects_are_known(): + assert _bounds_unknown((0, 0, 640, 29)) is False + assert _bounds_unknown((820, 433, 620, 19)) is False + + +def test_malformed_bounds_fail_open(): + assert _bounds_unknown(None) is False + assert _bounds_unknown(("x", 0, 0, 0)) is False + + +def test_json_bounds_nulled_for_zero_rect(): + out = _element_to_dict(_el((0, 0, 0, 0))) + assert out["bounds"] is None + assert out["index"] == 2 # index targeting stays intact + + +def test_json_bounds_preserved_for_real_rect(): + out = _element_to_dict(_el((10, 20, 30, 40))) + assert out["bounds"] == [10, 20, 30, 40] + + +def test_summary_line_says_bounds_unknown(): + lines = _format_elements([_el((0, 0, 0, 0))]) + assert "bounds-unknown" in lines[0] + assert "click by element index" in lines[0] + assert "(0, 0, 0, 0)" not in lines[0] + + +def test_summary_line_normal_for_real_rect(): + lines = _format_elements([_el((10, 20, 30, 40))]) + assert "bounds-unknown" not in lines[0] + assert "(10, 20, 30, 40)" in lines[0] diff --git a/tools/computer_use/tool.py b/tools/computer_use/tool.py index a77de3cd26..93dc89dfaf 100644 --- a/tools/computer_use/tool.py +++ b/tools/computer_use/tool.py @@ -1501,11 +1501,26 @@ def _maybe_follow_capture( return json.dumps(data) +def _bounds_unknown(bounds) -> bool: + """True when the AX tree reported no real geometry for an element. + + KDE/Qt apps commonly report ``[0, 0, 0, 0]`` for elements that are + perfectly clickable by index (live QA, Aug 2026: all of kcalc's radio + buttons). Serializing that as a plausible-looking rect invites a model + to derive ``coordinate=[0, 0]`` from it and click the screen corner. + """ + try: + return all(int(v) == 0 for v in bounds) + except (TypeError, ValueError): + return False + + def _format_elements(elements: List[UIElement], max_lines: int = 40) -> List[str]: out: List[str] = [] for e in elements[:max_lines]: label = e.label.replace("\n", " ")[:60] - out.append(f" #{e.index} {e.role} {label!r} @ {e.bounds}" + where = "@ bounds-unknown (click by element index)" if _bounds_unknown(e.bounds) else f"@ {e.bounds}" + out.append(f" #{e.index} {e.role} {label!r} {where}" + (f" [{e.app}]" if e.app else "")) if len(elements) > max_lines: out.append(f" ... +{len(elements) - max_lines} more (call capture with app= to narrow)") @@ -1673,7 +1688,9 @@ def _element_to_dict(e: UIElement) -> Dict[str, Any]: "index": e.index, "role": e.role, "label": label, - "bounds": list(e.bounds), + # A zero rect is "geometry unknown", not a position — null it so no + # coordinate= is ever derived from it. The element index still works. + "bounds": None if _bounds_unknown(e.bounds) else list(e.bounds), "app": e.app, } if truncated: