fix(computer-use): zero-rect AX bounds serialize as unknown, not a position

KDE/Qt apps report [0,0,0,0] bounds for elements that are perfectly
clickable by index (live QA: all 50 of kcalc's zero-rect elements,
including every radio button). Serializing that as a plausible rect
invites a model to derive coordinate=[0,0] and click the screen corner.

- _element_to_dict: zero rect -> bounds: null
- _format_elements: '@ bounds-unknown (click by element index)' instead
  of the fake rect in the summary line
- malformed bounds fail open (unchanged serialization)

Live-proven on real kcalc (cua-driver 0.20.0): 50 elements now null, 0
zero-rect leftovers, summary annotated, real rects preserved, and a
null-bounds radio button still clicks fine by index.
This commit is contained in:
Teknium
2026-08-15 22:28:10 -07:00
parent 1d5fc2bc0b
commit 9b9dd6eeab
2 changed files with 73 additions and 2 deletions
@@ -0,0 +1,54 @@
"""Zero-rect AX bounds must read as 'unknown', never as a clickable position.
Live QA (Aug 2026): KDE/Qt apps report [0,0,0,0] for elements that are
perfectly clickable by index (all of kcalc's radio buttons). A
plausible-looking zero rect invites coordinate=[0,0] derivation.
"""
from tools.computer_use.backend import UIElement
from tools.computer_use.tool import (
_bounds_unknown,
_element_to_dict,
_format_elements,
)
def _el(bounds, index=2, role="radio button", label="Deg"):
return UIElement(index=index, role=role, label=label, bounds=tuple(bounds), app="")
def test_zero_rect_is_unknown():
assert _bounds_unknown((0, 0, 0, 0)) is True
def test_real_rects_are_known():
assert _bounds_unknown((0, 0, 640, 29)) is False
assert _bounds_unknown((820, 433, 620, 19)) is False
def test_malformed_bounds_fail_open():
assert _bounds_unknown(None) is False
assert _bounds_unknown(("x", 0, 0, 0)) is False
def test_json_bounds_nulled_for_zero_rect():
out = _element_to_dict(_el((0, 0, 0, 0)))
assert out["bounds"] is None
assert out["index"] == 2 # index targeting stays intact
def test_json_bounds_preserved_for_real_rect():
out = _element_to_dict(_el((10, 20, 30, 40)))
assert out["bounds"] == [10, 20, 30, 40]
def test_summary_line_says_bounds_unknown():
lines = _format_elements([_el((0, 0, 0, 0))])
assert "bounds-unknown" in lines[0]
assert "click by element index" in lines[0]
assert "(0, 0, 0, 0)" not in lines[0]
def test_summary_line_normal_for_real_rect():
lines = _format_elements([_el((10, 20, 30, 40))])
assert "bounds-unknown" not in lines[0]
assert "(10, 20, 30, 40)" in lines[0]
+19 -2
View File
@@ -1501,11 +1501,26 @@ def _maybe_follow_capture(
return json.dumps(data)
def _bounds_unknown(bounds) -> bool:
"""True when the AX tree reported no real geometry for an element.
KDE/Qt apps commonly report ``[0, 0, 0, 0]`` for elements that are
perfectly clickable by index (live QA, Aug 2026: all of kcalc's radio
buttons). Serializing that as a plausible-looking rect invites a model
to derive ``coordinate=[0, 0]`` from it and click the screen corner.
"""
try:
return all(int(v) == 0 for v in bounds)
except (TypeError, ValueError):
return False
def _format_elements(elements: List[UIElement], max_lines: int = 40) -> List[str]:
out: List[str] = []
for e in elements[:max_lines]:
label = e.label.replace("\n", " ")[:60]
out.append(f" #{e.index} {e.role} {label!r} @ {e.bounds}"
where = "@ bounds-unknown (click by element index)" if _bounds_unknown(e.bounds) else f"@ {e.bounds}"
out.append(f" #{e.index} {e.role} {label!r} {where}"
+ (f" [{e.app}]" if e.app else ""))
if len(elements) > max_lines:
out.append(f" ... +{len(elements) - max_lines} more (call capture with app= to narrow)")
@@ -1673,7 +1688,9 @@ def _element_to_dict(e: UIElement) -> Dict[str, Any]:
"index": e.index,
"role": e.role,
"label": label,
"bounds": list(e.bounds),
# A zero rect is "geometry unknown", not a position — null it so no
# coordinate= is ever derived from it. The element index still works.
"bounds": None if _bounds_unknown(e.bounds) else list(e.bounds),
"app": e.app,
}
if truncated: