From bbd3462e25f47e6b0cfab8d0af8422f2230ee439 Mon Sep 17 00:00:00 2001 From: weisiwu <2436887475@qq.com> Date: Thu, 6 Aug 2026 00:34:41 +0800 Subject: [PATCH] fix(computer-use): merge refs+content_refs in _ref_map for cua-driver 0.17 cua-driver >= 0.17 splits the semantic_v2 snapshot payload: action-bearing refs live in the `refs` array while `content_refs` carries every node with EMPTY action lists. _ref_map only absorbed content_refs, so every click/pointer/type ref was registered with no declared actions and all typed-browser mutations failed with browser_ref_stale. Merge refs + content_refs + snapshot.refs with set union so action info is never dropped by an empty content entry. Verified against the 0.17 split format, the legacy refs-only format, the transitional dict format, and the snapshot.refs fallback. --- tools/computer_use/browser_route.py | 60 ++++++++++++++++------------- 1 file changed, 34 insertions(+), 26 deletions(-) diff --git a/tools/computer_use/browser_route.py b/tools/computer_use/browser_route.py index 9bdb9e1015..c6bbd77b07 100644 --- a/tools/computer_use/browser_route.py +++ b/tools/computer_use/browser_route.py @@ -60,36 +60,44 @@ def _ref_map(payload: Dict[str, Any]) -> Dict[str, Set[str]]: cua-driver has emitted both mapping and list representations while the semantic snapshot contract evolved. Accept both without weakening the capability rule: a ref with no declared action remains readable only. + + cua-driver >= 0.17 splits the semantic_v2 payload: action-bearing refs + live in the ``refs`` array while ``content_refs`` carries every node + with empty action lists. Merge both sources (plus the legacy + snapshot.refs forms) so click/pointer/type refs stay usable. """ normalized: Dict[str, Set[str]] = {} - snapshot = payload.get("snapshot") - # semantic_v2 carries the authoritative action-bearing entries in - # ``content_refs``; some transitional builds also emitted a ``refs`` list - # or map. Prefer the richer live shape, then accept both older forms. - raw = payload.get("content_refs") - if not raw: - raw = payload.get("refs") - if raw is None and isinstance(snapshot, dict): - raw = snapshot.get("refs") - if isinstance(raw, dict): - entries: Iterable[tuple[Optional[str], Any]] = raw.items() - elif isinstance(raw, list): - entries = ((None, item) for item in raw) - else: - entries = () - for key, value in entries: - if isinstance(value, dict): - ref = value.get("ref") or key - actions = value.get("actions") + def absorb(raw: Any) -> None: + if isinstance(raw, dict): + entries: Iterable[tuple[Optional[str], Any]] = raw.items() + elif isinstance(raw, list): + entries = ((None, item) for item in raw) else: - ref = key - actions = None - if not isinstance(ref, str) or not ref: - continue - normalized[ref] = { - action for action in (actions or []) if isinstance(action, str) - } + return + for key, value in entries: + if isinstance(value, dict): + ref = value.get("ref") or key + actions = value.get("actions") + else: + ref = key + actions = None + if not isinstance(ref, str) or not ref: + continue + action_set = { + action for action in (actions or []) if isinstance(action, str) + } + # Merge, never drop: content_refs entries carry empty action + # lists and must not clobber the same ref declared in ``refs``. + normalized[ref] = normalized.get(ref, set()) | action_set + + # 0.17 authoritative action refs; older builds emit only ``refs``; some + # transitional builds emit a mapping. Absorb every shape. + absorb(payload.get("refs")) + absorb(payload.get("content_refs")) + snapshot = payload.get("snapshot") + if isinstance(snapshot, dict): + absorb(snapshot.get("refs")) return normalized