Teknium
403 lines
15 KiB
Python
403 lines
15 KiB
Python
"""Pure parsing helpers for the cua-driver backend: MCP result flattening,
|
|
``list_windows`` / ``get_window_state`` payload normalisation, key combos.
|
|
|
|
No I/O, no module state — everything here is a function of its inputs, which
|
|
is what makes it safe to share between the MCP and CLI transports.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import re
|
|
from typing import Any, Dict, List, Optional, Tuple
|
|
|
|
from tools.computer_use.backend import ActionResult, UIElement
|
|
|
|
_MISSING = object()
|
|
|
|
# Linux/X11 can surface GNOME Shell / desktop backdrop windows before real app
|
|
# windows with no useful z-order. They are targetable X11 windows but capture
|
|
# as empty through get_window_state, so default app capture must skip them.
|
|
_NON_APP_WINDOW_TITLE_PREFIXES = (
|
|
"@!", # GNOME Shell background/monitor helper windows
|
|
"Desktop",
|
|
"gnome-shell",
|
|
"GNOME Shell",
|
|
)
|
|
|
|
_ELEMENT_LINE_RE = re.compile(
|
|
r'^\s*(?:-\s+)?\[(\d+)\]\s+(\w+)'
|
|
r'(?:'
|
|
r'\s*=\s*"([^"]*)"' # = "value"
|
|
r'|\s+"([^"]*)"' # "value"
|
|
r'|\s+\((?!\d+\))([^)]*)\)' # (value) but not a pure-digit (order) number
|
|
r')?'
|
|
r'(?:\s+(?:\(\d+\)\s+)?id=([^\s\[\]]+))?', # optional id=value (after an optional (order))
|
|
re.MULTILINE,
|
|
)
|
|
"""Element line of the get_window_state AX-tree markdown.
|
|
|
|
cua-driver renders each actionable node as ``[N] AXRole`` followed by a label
|
|
in one of four forms — ``= "value"``, ``"quoted"``, ``(paren)``, ``id=Label``
|
|
(optionally after an ``(order)`` number). A parenthesised pure-digit group is
|
|
an ORDER index, not a label, and is excluded so the id= label wins. Group 1 is
|
|
the index, group 2 the role, groups 3-6 the label in whichever form matched.
|
|
"""
|
|
|
|
|
|
def _mcp_field(obj, snake: str, camel: str, default=None):
|
|
"""Read an MCP model field across the 1.x -> 2.x rename.
|
|
|
|
mcp 2.0 exposes snake_case attributes and keeps camelCase only as a
|
|
serialization alias, so ``getattr(result, "isError", False)`` reads False
|
|
for every result on 2.x and a denied call would look like a success.
|
|
Deliberately duplicated from ``tools.mcp_tool.mcp_field`` so computer_use
|
|
never loads the much larger config-driven MCP client module.
|
|
"""
|
|
value = getattr(obj, snake, _MISSING)
|
|
if value is not _MISSING:
|
|
return value
|
|
value = getattr(obj, camel, _MISSING)
|
|
return default if value is _MISSING else value
|
|
|
|
|
|
def _action_result_from(
|
|
name: str,
|
|
ok: bool,
|
|
message: str,
|
|
meta: Dict[str, Any],
|
|
structured: Dict[str, Any],
|
|
*,
|
|
requested_delivery: Optional[str] = None,
|
|
) -> ActionResult:
|
|
"""Build an ActionResult, lifting cua-driver's structured verdict.
|
|
|
|
structuredContent is canonical, the flattened ``meta`` copy the fallback.
|
|
Every structured field is additive: a driver that omits one leaves the
|
|
attribute ``None`` so old drivers see unchanged behavior.
|
|
"""
|
|
sc = structured if isinstance(structured, dict) else {}
|
|
|
|
def _pick(key: str) -> Any:
|
|
return sc.get(key) if key in sc else meta.get(key)
|
|
|
|
def _typed(value: Any, typ) -> Any:
|
|
return value if isinstance(value, typ) else None
|
|
|
|
return ActionResult(
|
|
ok=ok,
|
|
action=name,
|
|
message=message,
|
|
meta=meta,
|
|
verified=_typed(_pick("verified"), bool),
|
|
effect=_typed(_pick("effect"), str),
|
|
escalation=_typed(_pick("escalation"), dict),
|
|
path=_typed(_pick("path"), str),
|
|
degraded=_typed(_pick("degraded"), bool),
|
|
# What we asked for; the driver's `path` records the rung that ran.
|
|
delivery_mode=_typed(requested_delivery, str),
|
|
# Refusal/limitation code — drivers spell it "code" or "reason_code".
|
|
code=_typed(_pick("code") or _pick("reason_code"), str),
|
|
)
|
|
|
|
|
|
def _z_index_uninformative(windows: List[Dict[str, Any]]) -> bool:
|
|
"""True when every window shares the same z_index (common on Linux/X11)."""
|
|
if not windows:
|
|
return True
|
|
return len({w.get("z_index", 0) for w in windows}) <= 1
|
|
|
|
|
|
def _parse_xprop_net_active_window(stdout: str) -> Optional[int]:
|
|
"""Parse ``xprop -root _NET_ACTIVE_WINDOW`` stdout into a window id.
|
|
|
|
Accepts the ``window id # 0x...`` form, falling back to the first hex token.
|
|
"""
|
|
text = stdout or ""
|
|
match = re.search(r"window id # (0x[0-9a-fA-F]+)", text) or re.search(r"(0x[0-9a-fA-F]+)", text)
|
|
if not match:
|
|
return None
|
|
try:
|
|
return int(match.group(1), 16)
|
|
except ValueError:
|
|
return None
|
|
|
|
|
|
def _is_real_app_window(w: Dict[str, Any]) -> bool:
|
|
"""Return False for desktop/shell helper windows that capture as empty."""
|
|
title = w.get("title", "")
|
|
return not any(
|
|
title.startswith(p) or title.lower().startswith(p.lower())
|
|
for p in _NON_APP_WINDOW_TITLE_PREFIXES
|
|
)
|
|
|
|
|
|
def _parse_elements_from_tree(markdown: str) -> List[UIElement]:
|
|
"""Parse UIElements from get_window_state AX-tree markdown.
|
|
|
|
Last-resort fallback for drivers without ``structuredContent.elements``.
|
|
Bounds always come back ``(0, 0, 0, 0)`` — the markdown carries none —
|
|
which is fine for element-index clicks (the driver resolves the frame).
|
|
"""
|
|
return [
|
|
UIElement(
|
|
index=int(m.group(1)),
|
|
role=m.group(2),
|
|
# groups 3-6: value / quoted / paren / id= label (first non-None wins)
|
|
label=m.group(3) or m.group(4) or m.group(5) or m.group(6) or "",
|
|
bounds=(0, 0, 0, 0),
|
|
)
|
|
for m in _ELEMENT_LINE_RE.finditer(markdown)
|
|
]
|
|
|
|
|
|
def _parse_elements_from_structured(raw_elements: List[Dict[str, Any]]) -> List[UIElement]:
|
|
"""Read the canonical ``structuredContent.elements`` array.
|
|
|
|
Each entry has ``element_index``, ``role``, ``label`` and, when the AT-SPI /
|
|
AXFrame call returned usable bounds, ``frame`` ``{x, y, w, h}`` — so real
|
|
pixel bounds survive (the markdown path loses them). Malformed entries are
|
|
skipped rather than failing the whole walk.
|
|
"""
|
|
elements: List[UIElement] = []
|
|
for raw in raw_elements:
|
|
if not isinstance(raw, dict):
|
|
continue
|
|
idx = raw.get("element_index")
|
|
if not isinstance(idx, int):
|
|
continue
|
|
role = raw.get("role") if isinstance(raw.get("role"), str) else ""
|
|
label = raw.get("label") if isinstance(raw.get("label"), str) else ""
|
|
frame = raw.get("frame") if isinstance(raw.get("frame"), dict) else None
|
|
bounds: Tuple[int, int, int, int] = (0, 0, 0, 0)
|
|
if frame:
|
|
try:
|
|
bounds = tuple(int(frame.get(k, 0)) for k in ("x", "y", "w", "h")) # type: ignore[assignment]
|
|
except (TypeError, ValueError):
|
|
bounds = (0, 0, 0, 0)
|
|
# Opaque element_token (`s{snapshot_hex}:{index}`) — the driver owns
|
|
# the parse + LRU semantics; we treat it as a black-box string.
|
|
raw_token = raw.get("element_token")
|
|
elements.append(UIElement(
|
|
index=idx,
|
|
role=role,
|
|
label=label,
|
|
bounds=bounds,
|
|
element_token=raw_token if isinstance(raw_token, str) and raw_token else None,
|
|
))
|
|
return elements
|
|
|
|
|
|
def _image_dimensions_from_bytes(raw: bytes) -> Tuple[int, int]:
|
|
"""Best-effort PNG/JPEG dimension sniffing without extra dependencies."""
|
|
if raw.startswith(b"\x89PNG\r\n\x1a\n") and len(raw) >= 24:
|
|
width = int.from_bytes(raw[16:20], "big")
|
|
height = int.from_bytes(raw[20:24], "big")
|
|
if width > 0 and height > 0:
|
|
return width, height
|
|
|
|
if raw.startswith(b"\xff\xd8"):
|
|
i = 2
|
|
n = len(raw)
|
|
while i + 9 < n:
|
|
if raw[i] != 0xFF:
|
|
i += 1
|
|
continue
|
|
marker = raw[i + 1]
|
|
i += 2
|
|
if marker in {0xD8, 0xD9} or 0xD0 <= marker <= 0xD7:
|
|
continue
|
|
if i + 2 > n:
|
|
break
|
|
segment_len = int.from_bytes(raw[i:i + 2], "big")
|
|
if segment_len < 2 or i + segment_len > n:
|
|
break
|
|
if marker in {
|
|
0xC0, 0xC1, 0xC2, 0xC3, 0xC5, 0xC6, 0xC7,
|
|
0xC9, 0xCA, 0xCB, 0xCD, 0xCE, 0xCF,
|
|
}:
|
|
if segment_len >= 7:
|
|
height = int.from_bytes(raw[i + 3:i + 5], "big")
|
|
width = int.from_bytes(raw[i + 5:i + 7], "big")
|
|
if width > 0 and height > 0:
|
|
return width, height
|
|
break
|
|
i += segment_len
|
|
|
|
return 0, 0
|
|
|
|
|
|
def _split_tree_text(full_text: str) -> Tuple[str, str]:
|
|
"""Split get_window_state text into (summary_line, tree_markdown)."""
|
|
summary, _, tree = full_text.partition("\n")
|
|
return summary, tree
|
|
|
|
|
|
_MODIFIER_NAMES = frozenset({"cmd", "command", "shift", "option", "alt", "ctrl", "control", "fn"})
|
|
_KEY_ALIASES = {"command": "cmd", "alt": "option", "control": "ctrl"}
|
|
|
|
|
|
def _parse_key_combo(keys: str) -> Tuple[Optional[str], List[str]]:
|
|
"""Parse 'cmd+s' / 'ctrl-alt-t' into (key, modifiers); last non-modifier wins."""
|
|
modifiers: List[str] = []
|
|
key = None
|
|
for part in (p.strip().lower() for p in re.split(r'[+\-]', keys) if p.strip()):
|
|
normalized = _KEY_ALIASES.get(part, part)
|
|
if normalized in _MODIFIER_NAMES:
|
|
modifiers.append(normalized)
|
|
else:
|
|
key = part
|
|
return key, modifiers
|
|
|
|
|
|
def _extract_tool_result(mcp_result: Any) -> Dict[str, Any]:
|
|
"""Flatten an mcp CallToolResult into
|
|
``{data, images, image_mime_types, structuredContent, isError}``.
|
|
|
|
``data`` is the joined text parts (parsed as JSON when it looks like JSON);
|
|
``image_mime_types`` is parallel to ``images`` with ``""`` where the part
|
|
carried no mimeType (older drivers — callers then sniff the base64 prefix).
|
|
"""
|
|
data: Any = None
|
|
images: List[str] = []
|
|
image_mime_types: List[str] = []
|
|
# Identity, not truthiness: mocks/proxies synthesize truthy attributes.
|
|
is_error = _mcp_field(mcp_result, "is_error", "isError", False) is True
|
|
structured: Optional[Dict] = (
|
|
_mcp_field(mcp_result, "structured_content", "structuredContent") or None
|
|
)
|
|
text_chunks: List[str] = []
|
|
for part in getattr(mcp_result, "content", []) or []:
|
|
ptype = getattr(part, "type", None)
|
|
if ptype == "text":
|
|
text_chunks.append(getattr(part, "text", "") or "")
|
|
elif ptype == "image":
|
|
b64 = getattr(part, "data", None)
|
|
if b64:
|
|
images.append(b64)
|
|
image_mime_types.append(_mcp_field(part, "mime_type", "mimeType") or "")
|
|
if text_chunks:
|
|
joined = "\n".join(t for t in text_chunks if t)
|
|
try:
|
|
data = json.loads(joined) if joined.strip().startswith(("{", "[")) else joined
|
|
except json.JSONDecodeError:
|
|
data = joined
|
|
return {
|
|
"data": data,
|
|
"images": images,
|
|
"image_mime_types": image_mime_types,
|
|
"structuredContent": structured,
|
|
"isError": is_error,
|
|
}
|
|
|
|
|
|
def _image_from_tool_result(out: Dict[str, Any]) -> tuple[Optional[str], Optional[str]]:
|
|
"""Pull ``(b64, mime_type)`` out of a flattened tool result.
|
|
|
|
cua-driver delivers screenshots either as an MCP ``image`` part
|
|
(``out["images"]``) or as ``screenshot_png_b64`` inside structuredContent
|
|
(newer builds, and the CLI transport); checking both keeps capture()
|
|
robust when the driver moves the image between the two.
|
|
"""
|
|
images = out.get("images") or []
|
|
if images and images[0]:
|
|
mimes = out.get("image_mime_types") or []
|
|
return images[0], (mimes[0] if mimes and mimes[0] else None)
|
|
structured = out.get("structuredContent") or {}
|
|
b64 = structured.get("screenshot_png_b64") or structured.get("png_b64")
|
|
if b64:
|
|
return b64, (structured.get("screenshot_mime_type") or structured.get("mime_type") or None)
|
|
return None, None
|
|
|
|
|
|
def _positive_int(value: Any) -> Optional[int]:
|
|
"""Return a positive integer, rejecting booleans and malformed values."""
|
|
if isinstance(value, bool) or not isinstance(value, (int, str)):
|
|
return None
|
|
try:
|
|
parsed = int(value)
|
|
except ValueError:
|
|
return None
|
|
return parsed if parsed > 0 else None
|
|
|
|
|
|
def _is_placeholder_id(value: Any) -> bool:
|
|
"""True when *value* is a schema-filler id (``0`` / negative) rather than a target.
|
|
|
|
Some providers emit every optional integer zero-filled; treating that as a
|
|
targeting request would drop the caller's ``app=``. Non-numeric values are
|
|
NOT placeholders — they still reach the validation error.
|
|
"""
|
|
if isinstance(value, bool) or not isinstance(value, (int, str)):
|
|
return False
|
|
try:
|
|
return int(value) <= 0
|
|
except ValueError:
|
|
return False
|
|
|
|
|
|
def _ingest_windows(raw_windows: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
|
|
"""Normalise cua-driver ``list_windows`` entries, dropping unusable ones.
|
|
|
|
Every downstream call needs an integer ``pid`` and ``window_id``. On X11
|
|
the PID comes from the optional ``_NET_WM_PID`` property, so root/panel/
|
|
popup windows report ``pid: null`` — skip those instead of aborting the
|
|
whole enumeration. ``z_index``: higher = closer to front; Wayland's null
|
|
(undefined stacking) sorts lowest so real windows stay above the desktop.
|
|
"""
|
|
windows: List[Dict[str, Any]] = []
|
|
for w in raw_windows:
|
|
if not isinstance(w, dict): # untrusted compatibility envelopes
|
|
continue
|
|
pid_int = _positive_int(w.get("pid"))
|
|
window_id_int = _positive_int(w.get("window_id"))
|
|
if pid_int is None or window_id_int is None:
|
|
continue
|
|
z_raw = w.get("z_index")
|
|
app_name = w.get("app_name", "")
|
|
title = w.get("title", "")
|
|
windows.append({
|
|
"app_name": app_name if isinstance(app_name, str) else "",
|
|
"pid": pid_int,
|
|
"window_id": window_id_int,
|
|
# Only explicit False means off-screen; null (Linux 0.6.x) means unknown.
|
|
"off_screen": w.get("is_on_screen") is False,
|
|
"title": title if isinstance(title, str) else "",
|
|
"z_index": z_raw if isinstance(z_raw, (int, float)) and not isinstance(z_raw, bool) else 0,
|
|
})
|
|
return windows
|
|
|
|
|
|
def _first_nonempty_list(*containers: Any, keys: Tuple[str, ...]) -> List[Any]:
|
|
for container in containers:
|
|
if not isinstance(container, dict):
|
|
continue
|
|
for key in keys:
|
|
value = container.get(key)
|
|
if isinstance(value, list) and value:
|
|
return value
|
|
return []
|
|
|
|
|
|
def _windows_from_tool_result(out: Dict[str, Any]) -> List[Dict[str, Any]]:
|
|
"""Return list_windows payloads across cua-driver result shapes."""
|
|
structured = out.get("structuredContent")
|
|
if isinstance(structured, dict):
|
|
windows = structured.get("windows")
|
|
if isinstance(windows, list) and windows:
|
|
return windows
|
|
return _first_nonempty_list(out.get("data"), out, keys=("windows", "_legacy_windows"))
|
|
|
|
|
|
def _apps_from_windows(windows: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
|
|
apps: List[Dict[str, Any]] = []
|
|
seen: set[tuple[str, int]] = set()
|
|
for summary in _ingest_windows(windows):
|
|
name = summary["app_name"]
|
|
key = (name, summary["pid"])
|
|
if not name or key in seen:
|
|
continue
|
|
seen.add(key)
|
|
apps.append({"name": name, "pid": summary["pid"]})
|
|
return apps
|