fix(computer-use): zero-rect AX bounds serialize as unknown, not a position
KDE/Qt apps report [0,0,0,0] bounds for elements that are perfectly clickable by index (live QA: all 50 of kcalc's zero-rect elements, including every radio button). Serializing that as a plausible rect invites a model to derive coordinate=[0,0] and click the screen corner. - _element_to_dict: zero rect -> bounds: null - _format_elements: '@ bounds-unknown (click by element index)' instead of the fake rect in the summary line - malformed bounds fail open (unchanged serialization) Live-proven on real kcalc (cua-driver 0.20.0): 50 elements now null, 0 zero-rect leftovers, summary annotated, real rects preserved, and a null-bounds radio button still clicks fine by index.
This commit is contained in:
@@ -0,0 +1,54 @@
|
||||
"""Zero-rect AX bounds must read as 'unknown', never as a clickable position.
|
||||
|
||||
Live QA (Aug 2026): KDE/Qt apps report [0,0,0,0] for elements that are
|
||||
perfectly clickable by index (all of kcalc's radio buttons). A
|
||||
plausible-looking zero rect invites coordinate=[0,0] derivation.
|
||||
"""
|
||||
from tools.computer_use.backend import UIElement
|
||||
from tools.computer_use.tool import (
|
||||
_bounds_unknown,
|
||||
_element_to_dict,
|
||||
_format_elements,
|
||||
)
|
||||
|
||||
|
||||
def _el(bounds, index=2, role="radio button", label="Deg"):
|
||||
return UIElement(index=index, role=role, label=label, bounds=tuple(bounds), app="")
|
||||
|
||||
|
||||
def test_zero_rect_is_unknown():
|
||||
assert _bounds_unknown((0, 0, 0, 0)) is True
|
||||
|
||||
|
||||
def test_real_rects_are_known():
|
||||
assert _bounds_unknown((0, 0, 640, 29)) is False
|
||||
assert _bounds_unknown((820, 433, 620, 19)) is False
|
||||
|
||||
|
||||
def test_malformed_bounds_fail_open():
|
||||
assert _bounds_unknown(None) is False
|
||||
assert _bounds_unknown(("x", 0, 0, 0)) is False
|
||||
|
||||
|
||||
def test_json_bounds_nulled_for_zero_rect():
|
||||
out = _element_to_dict(_el((0, 0, 0, 0)))
|
||||
assert out["bounds"] is None
|
||||
assert out["index"] == 2 # index targeting stays intact
|
||||
|
||||
|
||||
def test_json_bounds_preserved_for_real_rect():
|
||||
out = _element_to_dict(_el((10, 20, 30, 40)))
|
||||
assert out["bounds"] == [10, 20, 30, 40]
|
||||
|
||||
|
||||
def test_summary_line_says_bounds_unknown():
|
||||
lines = _format_elements([_el((0, 0, 0, 0))])
|
||||
assert "bounds-unknown" in lines[0]
|
||||
assert "click by element index" in lines[0]
|
||||
assert "(0, 0, 0, 0)" not in lines[0]
|
||||
|
||||
|
||||
def test_summary_line_normal_for_real_rect():
|
||||
lines = _format_elements([_el((10, 20, 30, 40))])
|
||||
assert "bounds-unknown" not in lines[0]
|
||||
assert "(10, 20, 30, 40)" in lines[0]
|
||||
@@ -1501,11 +1501,26 @@ def _maybe_follow_capture(
|
||||
return json.dumps(data)
|
||||
|
||||
|
||||
def _bounds_unknown(bounds) -> bool:
|
||||
"""True when the AX tree reported no real geometry for an element.
|
||||
|
||||
KDE/Qt apps commonly report ``[0, 0, 0, 0]`` for elements that are
|
||||
perfectly clickable by index (live QA, Aug 2026: all of kcalc's radio
|
||||
buttons). Serializing that as a plausible-looking rect invites a model
|
||||
to derive ``coordinate=[0, 0]`` from it and click the screen corner.
|
||||
"""
|
||||
try:
|
||||
return all(int(v) == 0 for v in bounds)
|
||||
except (TypeError, ValueError):
|
||||
return False
|
||||
|
||||
|
||||
def _format_elements(elements: List[UIElement], max_lines: int = 40) -> List[str]:
|
||||
out: List[str] = []
|
||||
for e in elements[:max_lines]:
|
||||
label = e.label.replace("\n", " ")[:60]
|
||||
out.append(f" #{e.index} {e.role} {label!r} @ {e.bounds}"
|
||||
where = "@ bounds-unknown (click by element index)" if _bounds_unknown(e.bounds) else f"@ {e.bounds}"
|
||||
out.append(f" #{e.index} {e.role} {label!r} {where}"
|
||||
+ (f" [{e.app}]" if e.app else ""))
|
||||
if len(elements) > max_lines:
|
||||
out.append(f" ... +{len(elements) - max_lines} more (call capture with app= to narrow)")
|
||||
@@ -1673,7 +1688,9 @@ def _element_to_dict(e: UIElement) -> Dict[str, Any]:
|
||||
"index": e.index,
|
||||
"role": e.role,
|
||||
"label": label,
|
||||
"bounds": list(e.bounds),
|
||||
# A zero rect is "geometry unknown", not a position — null it so no
|
||||
# coordinate= is ever derived from it. The element index still works.
|
||||
"bounds": None if _bounds_unknown(e.bounds) else list(e.bounds),
|
||||
"app": e.app,
|
||||
}
|
||||
if truncated:
|
||||
|
||||
Reference in New Issue
Block a user