refactor(desktop-tools): consolidate preview + project, diet the desktop_ui suite (3,861 → 2,293 tok/call, −41%) (#97659)

* refactor(desktop-tools): consolidate preview(open/close/read) + project(create/switch/list), diet the desktop_ui suite — 3,861 -> 2,293 tok/call on desktop sessions (-41%)

* rename: preview -> desktop_preview, project -> desktop_project — namespace desktop-app tools against MCP/plugin name collisions

* test: sync remaining old-name pins — per-file registration import, GUI_TOOLS set, post-hook case read_preview -> desktop_preview action=read
This commit is contained in:
Teknium
2026-08-28 23:10:01 -07:00
committed by GitHub
parent 70c6d78cae
commit 217ab2f8df
23 changed files with 383 additions and 274 deletions
+16 -11
View File
@@ -108,7 +108,7 @@ def _ra():
AGENT_RUNTIME_POST_HOOK_TOOL_NAMES = frozenset(
{"todo", "session_search", "memory", "clarify", "read_terminal", "read_preview", "drive_preview", "annotate_preview", "read_window_below", "setup_mcp", "tour", "delegate_task"}
{"todo", "session_search", "memory", "clarify", "read_terminal", "desktop_preview", "drive_preview", "annotate_preview", "read_window_below", "setup_mcp", "tour", "delegate_task"}
)
@@ -3481,17 +3481,22 @@ def invoke_tool(agent, function_name: str, function_args: dict, effective_task_i
),
next_args,
)
elif function_name == "read_preview":
elif function_name == "desktop_preview":
def _execute(next_args: dict) -> Any:
from tools.read_preview_tool import read_preview_tool as _read_preview_tool
return _finish_agent_tool(
_read_preview_tool(
start=next_args.get("start"),
count=next_args.get("count"),
callback=getattr(agent, "read_preview_callback", None),
),
next_args,
)
# action=read needs the GUI callback (agent-level); open/close go
# through the registry handler like any other tool.
if (next_args.get("action") or "").strip() == "read":
from tools.read_preview_tool import read_preview_tool as _read_preview_tool
return _finish_agent_tool(
_read_preview_tool(
start=next_args.get("start"),
count=next_args.get("count"),
callback=getattr(agent, "read_preview_callback", None),
),
next_args,
)
from tools.preview_tool import _handle_preview
return _finish_agent_tool(_handle_preview(next_args), next_args)
elif function_name == "drive_preview":
def _execute(next_args: dict) -> Any:
from tools.drive_preview_tool import drive_preview_tool as _drive_preview_tool
+11 -8
View File
@@ -2242,14 +2242,17 @@ def execute_tool_calls_sequential(agent, assistant_message, messages: list, effe
tool_duration = time.time() - tool_start_time
if agent._should_emit_quiet_tool_messages():
agent._vprint(f" {_get_cute_tool_message_impl('read_terminal', function_args, tool_duration, result=function_result)}")
elif function_name == "read_preview":
elif function_name == "desktop_preview":
def _execute(next_args: dict) -> Any:
from tools.read_preview_tool import read_preview_tool as _read_preview_tool
return _read_preview_tool(
start=next_args.get("start"),
count=next_args.get("count"),
callback=getattr(agent, "read_preview_callback", None),
)
if (next_args.get("action") or "").strip() == "read":
from tools.read_preview_tool import read_preview_tool as _read_preview_tool
return _read_preview_tool(
start=next_args.get("start"),
count=next_args.get("count"),
callback=getattr(agent, "read_preview_callback", None),
)
from tools.preview_tool import _handle_preview
return _handle_preview(next_args)
function_result, function_args, middleware_trace, _execution_blocked, _execution_dispatched = _managed_values(_run_agent_tool_execution_middleware(
agent,
function_name=function_name,
@@ -2262,7 +2265,7 @@ def execute_tool_calls_sequential(agent, assistant_message, messages: list, effe
))
tool_duration = time.time() - tool_start_time
if agent._should_emit_quiet_tool_messages():
agent._vprint(f" {_get_cute_tool_message_impl('read_preview', function_args, tool_duration, result=function_result)}")
agent._vprint(f" {_get_cute_tool_message_impl('desktop_preview', function_args, tool_duration, result=function_result)}")
elif function_name == "drive_preview":
def _execute(next_args: dict) -> Any:
from tools.drive_preview_tool import drive_preview_tool as _drive_preview_tool
+1 -1
View File
@@ -2446,7 +2446,7 @@ class TestAgentRuntimePostHookOwnershipSync:
("memory", {"action": "view", "target": "memory"}),
("clarify", {"question": "Continue?"}),
("read_terminal", {}),
("read_preview", {}),
("desktop_preview", {"action": "read"}),
("drive_preview", {"action": "elements"}),
("annotate_preview", {"action": "clear"}),
("read_window_below", {}),
+6 -4
View File
@@ -17,11 +17,13 @@ def _reset_emitter():
def test_lives_in_the_gui_surface_toolset(monkeypatch):
"""Reaches a desktop client on ANY backend, including one with no
HERMES_DESKTOP in its environment (URL / cloud gateways)."""
import tools.preview_tool # noqa: F401 — registers desktop_preview
"""Consolidated (#95681): this module's tool became an action of the
single `desktop_preview` tool in desktop_ui; the old registration is gone and
`preview` reaches a desktop client on ANY backend (no env gate)."""
monkeypatch.delenv("HERMES_DESKTOP", raising=False)
entry = registry.get_entry("close_preview")
assert registry.get_entry("close_preview") is None
entry = registry.get_entry("desktop_preview")
assert entry is not None
assert entry.toolset == "desktop_ui"
assert entry.check_fn is None
+113
View File
@@ -0,0 +1,113 @@
"""Desktop tool consolidation + diet (#95681, maintainer-directed).
preview = open/close/read as one action tool (576 -> ~235); project =
create/switch/list as one (244 -> ~155). Old names are GONE from the
toolsets (desktop-only tools; no long-transcript compat needed). The
preview read action still routes through the agent-level GUI callback.
"""
import json
import os
import sys
import unittest
from unittest.mock import patch
sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", ".."))
class TestConsolidatedToolsets(unittest.TestCase):
def test_old_names_gone_new_names_present(self):
from toolsets import TOOLSETS
ui = TOOLSETS["desktop_ui"]["tools"]
self.assertIn("desktop_preview", ui)
for dead in ("open_preview", "close_preview", "read_preview"):
self.assertNotIn(dead, ui)
proj = TOOLSETS["project"]["tools"]
self.assertEqual(proj, ["desktop_project"])
def test_registry_serves_only_new_names(self):
from model_tools import get_tool_definitions
names = {
t["function"]["name"]
for t in get_tool_definitions(
quiet_mode=True, enabled_toolsets=["desktop_ui", "project"]
)
}
self.assertIn("desktop_preview", names)
self.assertIn("desktop_project", names)
for dead in (
"open_preview", "close_preview", "read_preview",
"project_create", "project_switch", "project_list",
):
self.assertNotIn(dead, names)
class TestPreviewHandler(unittest.TestCase):
def test_open_and_close_route_through_desktop_ui(self):
from tools import preview_tool
sent = []
with patch("tools.desktop_ui.emit", side_effect=lambda ev, p: sent.append((ev, p)) or True):
r = json.loads(preview_tool._handle_preview({"action": "open", "url": "www.cnn.com"}))
self.assertTrue(r["success"])
self.assertEqual(r["url"], "https://www.cnn.com") # normalizer kept
r = json.loads(preview_tool._handle_preview({"action": "close"}))
self.assertTrue(r["success"])
self.assertEqual([e for e, _ in sent], ["preview.open", "preview.close"])
def test_read_outside_desktop_session_teaches(self):
from tools import preview_tool
r = json.loads(preview_tool._handle_preview({"action": "read"}))
self.assertFalse(r.get("success", False))
def test_unknown_action_teaches(self):
from tools import preview_tool
r = json.loads(preview_tool._handle_preview({"action": "zap"}))
self.assertIn("open, close, read", r.get("error", ""))
class TestProjectHandler(unittest.TestCase):
def test_dispatch_shapes(self):
import tools.project_tools as pt
with patch.object(pt, "project_list", return_value='{"success": true}') as pl:
pt._handle_project({"action": "list"})
pl.assert_called_once()
with patch.object(pt, "project_create", return_value='{"success": true}') as pc:
pt._handle_project({"action": "create", "name": "X", "path": "C:/tmp"})
pc.assert_called_once_with(name="X", path="C:/tmp", task_id=None)
with patch.object(pt, "project_switch", return_value='{"success": true}') as ps:
pt._handle_project({"action": "switch", "name": "aurora"})
ps.assert_called_once_with(project="aurora", task_id=None)
r = json.loads(pt._handle_project({"action": "bogus"}))
self.assertFalse(r["success"])
class TestDietBudget(unittest.TestCase):
def test_desktop_surface_under_budget(self):
"""The 15-tool surface serialized to ~15.5K chars (≈3,861 tok);
consolidation + diet brought the (now 11-tool) surface to ~9.2K
chars (≈2,293 tok). Guard: stay under 10.5K chars (≈2,600 tok) —
a bloat regression guard, not a snapshot pin. Chars, not tokens:
tiktoken is not a repo dependency, and the chars/tokens ratio for
these schemas is stable (~4.0)."""
from model_tools import get_tool_definitions
targets = {
"drive_preview", "tour", "annotate_preview", "setup_mcp", "tip",
"desktop_preview", "desktop_project", "read_window_below", "apply_layout",
"read_terminal", "focus_pane",
}
total = 0
for t in get_tool_definitions(quiet_mode=True, enabled_toolsets=["desktop_ui", "project"]):
f = t["function"]
if f["name"] in targets:
total += len(json.dumps(f, separators=(",", ":"), ensure_ascii=False))
self.assertLess(total, 10_500, f"desktop tool surface regressed to {total} chars")
if __name__ == "__main__":
unittest.main()
+6 -4
View File
@@ -17,11 +17,13 @@ def _reset_emitter():
def test_lives_in_the_gui_surface_toolset(monkeypatch):
"""Reaches a desktop client on ANY backend, including one with no
HERMES_DESKTOP in its environment (URL / cloud gateways)."""
import tools.preview_tool # noqa: F401 — registers desktop_preview
"""Consolidated (#95681): this module's tool became an action of the
single `desktop_preview` tool in desktop_ui; the old registration is gone and
`preview` reaches a desktop client on ANY backend (no env gate)."""
monkeypatch.delenv("HERMES_DESKTOP", raising=False)
entry = registry.get_entry("open_preview")
assert registry.get_entry("open_preview") is None
entry = registry.get_entry("desktop_preview")
assert entry is not None
assert entry.toolset == "desktop_ui"
assert entry.check_fn is None
+6 -3
View File
@@ -7,10 +7,13 @@ from tools.registry import registry
def test_lives_in_the_gui_surface_toolset(monkeypatch):
"""Mirrors read_terminal: scoped by toolset, not by the backend's env."""
import tools.preview_tool # noqa: F401 — registers desktop_preview
"""Consolidated (#95681): this module's tool became an action of the
single `desktop_preview` tool in desktop_ui; the old registration is gone and
`preview` reaches a desktop client on ANY backend (no env gate)."""
monkeypatch.delenv("HERMES_DESKTOP", raising=False)
entry = registry.get_entry("read_preview")
assert registry.get_entry("read_preview") is None
entry = registry.get_entry("desktop_preview")
assert entry is not None
assert entry.toolset == "desktop_ui"
assert entry.check_fn is None
@@ -19,12 +19,10 @@ from toolsets import TOOLSETS, resolve_toolset
GUI_TOOLS = {
"annotate_preview",
"close_preview",
"desktop_preview",
"drive_preview",
"close_terminal",
"focus_pane",
"open_preview",
"read_preview",
"read_terminal",
"read_window_below",
"react_to_message",
+12 -24
View File
@@ -89,22 +89,14 @@ def annotate_preview_tool(
ANNOTATE_PREVIEW_SCHEMA = {
"name": "annotate_preview",
"description": (
"Draw a lasting mark on the page open in the in-app browser / preview "
"pane of the Hermes desktop GUI. Everything drive_preview draws as it "
"works fades on its own; an annotation STAYS until you remove it, so "
"this is how you point at something. Use it to show the user what you "
"found ('here are the three cheapest'), flag what you are about to "
"change before you change it, or keep your place while you work "
"elsewhere on the page. Address elements by the same refs "
"drive_preview action='elements' hands back. action='add' outlines an "
"element and gives it an optional short label; 'hold' freezes the WHOLE "
"visible field at once — every element the page offers, outlined and "
"named — which is the "
"picture drive_preview flashes as it works, made to stay; 'remove' "
"takes one down; 'clear' takes them all down. Annotations follow their element "
"as the page scrolls and disappear if it does, so a navigation clears "
"them for you. Keep labels to a word or two — they are drawn on the "
"page, not read aloud."
"Leave a LASTING mark on the preview-pane page (drive_preview's own "
"marks fade; annotations stay until removed) — point at findings, "
"flag what you're about to change, keep your place. Use the refs "
"from drive_preview action='elements'. add: outline one element "
"(optional short label — a word or two, drawn on the page). hold: "
"freeze the whole visible field, every element outlined and named. "
"remove/clear: take one/all down. Marks follow their element on "
"scroll; navigation clears them."
),
"parameters": {
"type": "object",
@@ -112,23 +104,19 @@ ANNOTATE_PREVIEW_SCHEMA = {
"action": {
"type": "string",
"enum": list(ACTIONS),
"description": (
"'add' marks one element, 'hold' freezes the whole visible "
"field, 'remove' takes one down, 'clear' takes them all "
"down. Defaults to 'add'."
),
"description": "Defaults to 'add'.",
},
"ref": {
"type": "string",
"description": "Element reference from drive_preview action='elements' (e.g. 'btn-sign-in').",
"description": "Ref from drive_preview elements.",
},
"selector": {
"type": "string",
"description": "CSS selector, as a fallback when no ref fits. Prefer ref.",
"description": "CSS selector fallback. Prefer ref.",
},
"label": {
"type": "string",
"description": "Short caption drawn on the mark, e.g. 'cheapest'. Optional.",
"description": "Optional caption, e.g. 'cheapest'.",
},
},
"required": [],
+4 -7
View File
@@ -44,13 +44,10 @@ def apply_layout_tool(preset: str) -> str:
APPLY_LAYOUT_SCHEMA = {
"name": "apply_layout",
"description": (
"Apply a saved layout preset to the Hermes desktop app when the user asks to "
"rearrange the workspace — e.g. \"set up my layout for coding\", \"give me a "
"focused view\", \"put the terminal front and center\". Built-in presets: "
"default (chat + sidebars), focus (chat only), terminal-deck (terminal "
"forward), quad (four zones). Plugin and user-saved presets are addressed by "
"their id. To reveal a single pane without rearranging everything, use "
"focus_pane instead."
"Apply a saved layout preset to the Hermes desktop app when the user "
"asks to rearrange the workspace. Built-ins: default (chat + "
"sidebars), focus (chat only), terminal-deck, quad; plugin/user "
"presets by id. To reveal ONE pane, use focus_pane instead."
),
"parameters": {
"type": "object",
+2 -7
View File
@@ -55,10 +55,5 @@ CLOSE_PREVIEW_SCHEMA = {
}
registry.register(
name="close_preview",
toolset="desktop_ui",
schema=CLOSE_PREVIEW_SCHEMA,
handler=lambda args, **kw: close_preview_tool(url=args.get("url") or ""),
emoji="🖼️",
)
# Registration removed: consolidated into the `preview` tool (#95681);
# this module keeps its functions for the preview_tool.
+32 -47
View File
@@ -126,44 +126,29 @@ def drive_preview_tool(
ACT_PREVIEW_SCHEMA = {
"name": "drive_preview",
# Dieted (#95681): world-building compressed; response-shape teaching
# kept only where skipping it causes wasted calls (delta semantics,
# rebound refs, strobe's burst) — those are pre-effect: a model that
# doesn't know them re-reads pages or loops strobe.
"description": (
"Interact with the page open in the in-app browser / preview pane of "
"the Hermes desktop GUI — the pane open_preview opens beside this "
"chat. This is how you USE a web app the user is looking at: log in, "
"fill a form, click through a flow, page a long document. ALWAYS call "
"action='elements' first to get the current inventory of clickable and "
"typable things — each carries a ref like 'btn-sign-in' or 'inp-email' "
"plus its role, label, and value — then act with that ref instead of "
"guessing a selector. A ref keeps working for as long as the page is "
"open, INCLUDING across a re-render that rebuilds the element, so hold "
"onto the ones you were given. "
"Every action answers with the live url/title plus what moved: the "
"first look at a page returns the full 'elements' inventory, and after "
"that you get a 'delta' instead — 'added' entries in full, 'changed' "
"entries carrying only the ref and whichever of label/value/disabled "
"actually moved, 'removed' and 'rebound' as bare ref lists, and 'same' "
"counting the refs that held. A 'rebound' ref needs NO action from you; it "
"means the page rebuilt that element and your ref already follows it. "
"Anything not mentioned in a delta is unchanged, so do not re-read the "
"page to check. Only a navigation invalidates refs; when told they are "
"stale, call elements again. The mouse "
"and keyboard are real: the pointer travels to its target and the page "
"sees genuine input, so hover menus open and hover-only controls work. "
"Actions: 'elements' (inventory), 'click', 'hover' (move the pointer "
"onto something and leave it there — use it to open a dropdown or "
"reveal a tooltip before clicking inside it), 'type' (set a field's "
"text; submit=true also presses Enter and submits the form), 'scroll' "
"(the page, or a ref'd scrollable), 'press' (a named key), 'strobe' "
"(touch nothing — just rattle the highlight through the page again; "
"'elements' already does this once, so reach for it only when asked to "
"flick, flash, or bounce around the page some more, and note one call "
"runs a whole multi-second burst, so never loop it per element), and "
"'back'/'forward'/'reload' for history. The pane draws every move as "
"it happens so the user can follow along; those marks fade on their "
"own, and annotate_preview is how you leave one up on purpose. Use "
"read_preview when you only "
"need the page's text, and the browser_* tools when the work belongs "
"in a separate automated browser rather than the user's own pane."
"Use the web page open in the desktop preview pane (the one "
"`desktop_preview` opens): log in, fill forms, click through flows. ALWAYS "
"start with action='elements' — it inventories clickable/typable "
"things as refs ('btn-sign-in') with role/label/value; act by ref, "
"not guessed selectors. Refs survive re-renders and only die on "
"navigation (you'll be told they're stale — call elements again). "
"After the first full inventory, actions answer with a DELTA: "
"'added' in full, 'changed' as ref + moved fields, 'removed'/"
"'rebound' as ref lists ('rebound' needs nothing from you — the ref "
"already follows the rebuilt element). Anything unmentioned is "
"unchanged; do not re-read to check. Input is real (pointer travels, "
"hover menus open). Actions: elements, click, hover (park the "
"pointer — opens dropdowns before clicking in), type (submit=true "
"also presses Enter), scroll, press, strobe (visual flourish only — "
"one call runs a multi-second burst; never loop it), back/forward/"
"reload. Moves draw live and fade; annotate_preview leaves a lasting "
"mark. Page text only: desktop_preview action=read. Separate automated "
"browser: browser_* tools."
),
"parameters": {
"type": "object",
@@ -171,41 +156,41 @@ ACT_PREVIEW_SCHEMA = {
"action": {
"type": "string",
"enum": list(ACTIONS),
"description": "What to do. Start with 'elements'.",
"description": "Start with 'elements'.",
},
"ref": {
"type": "string",
"description": "Element reference from any earlier elements call (e.g. 'btn-sign-in'). Good until the page navigates.",
"description": "Element ref from an earlier elements call.",
},
"selector": {
"type": "string",
"description": "CSS selector, as a fallback when no ref fits. Prefer ref.",
"description": "CSS selector fallback. Prefer ref.",
},
"text": {"type": "string", "description": "For 'type': the text to enter."},
"text": {"type": "string", "description": "type: the text."},
"submit": {
"type": "boolean",
"description": "For 'type': press Enter and submit the owning form afterwards.",
"description": "type: press Enter + submit the form after.",
},
"key": {
"type": "string",
"description": "For 'press': the key name, e.g. 'Enter', 'Escape', 'ArrowDown'.",
"description": "press: key name ('Enter', 'Escape', 'ArrowDown').",
},
"amount": {
"type": "integer",
"description": "For 'scroll': pixels to scroll (negative scrolls up). Defaults to about one screen.",
"description": "scroll: pixels (negative = up; default ~one screen).",
},
"to": {
"type": "string",
"enum": list(SCROLL_TO),
"description": "For 'scroll': jump to the top or bottom instead of a distance.",
"description": "scroll: jump to top/bottom instead.",
},
"max": {
"type": "integer",
"description": "For 'elements': cap the inventory. Defaults to the per-call maximum.",
"description": "elements: cap the inventory.",
},
"full": {
"type": "boolean",
"description": "For 'elements': re-read the whole page instead of a delta. Rarely needed.",
"description": "elements: full re-read instead of a delta. Rarely needed.",
},
},
"required": ["action"],
+3 -6
View File
@@ -36,12 +36,9 @@ def focus_pane_tool(pane: str) -> str:
FOCUS_PANE_SCHEMA = {
"name": "focus_pane",
"description": (
"Reveal and focus a pane in the Hermes desktop app when the user asks to "
"see it — e.g. \"show me the terminal\", \"open the file browser\", \"show "
"the diff\". Panes: chat (the conversation), files (project file browser), "
"terminal (embedded shell), review (git diff), sessions (the session list). "
"To show a URL or file in the preview pane, use open_preview; to close it, "
"use close_preview."
"Reveal and focus a Hermes desktop pane when the user asks to see it: "
"chat, files, terminal, review (git diff), or sessions. For URLs/"
"files use the desktop_preview tool instead."
),
"parameters": {
"type": "object",
+2 -7
View File
@@ -84,10 +84,5 @@ OPEN_PREVIEW_SCHEMA = {
}
registry.register(
name="open_preview",
toolset="desktop_ui",
schema=OPEN_PREVIEW_SCHEMA,
handler=lambda args, **kw: open_preview_tool(url=args.get("url", ""), label=args.get("label", "")),
emoji="🖼️",
)
# Registration removed: consolidated into the `preview` tool (#95681);
# this module keeps its functions for the preview_tool.
+85
View File
@@ -0,0 +1,85 @@
#!/usr/bin/env python3
"""The `desktop_preview` tool — the preview pane beside the chat, as ONE tool.
Consolidation (#95681, maintainer-directed): open_preview, close_preview,
and read_preview each re-taught "the preview pane beside the chat" world;
one action enum states it once (576 -> ~210 tok). The read action keeps
its agent-callback dispatch (agent_runtime_helpers routes action=read
through agent.read_preview_callback, same as read_preview did).
Lives in the ``desktop_ui`` toolset — desktop-app sessions only.
"""
import json
from tools import desktop_ui
from tools.open_preview_tool import _normalize_target, open_preview_tool
from tools.registry import registry, tool_error
def preview_open(url: str, label: str = "") -> str:
return open_preview_tool(url=url, label=label)
def preview_close(url: str = "") -> str:
target = _normalize_target(url or "")
try:
ok = desktop_ui.emit("preview.close", {"url": target})
except Exception as exc: # noqa: BLE001
return tool_error(f"Failed to close the preview: {exc}")
if not ok:
return tool_error("The preview pane is only available in the Hermes desktop app.")
return json.dumps({"success": True, "closed": target or "all"}, ensure_ascii=False)
def _handle_preview(args, **kw):
"""Non-read actions only: action=read is dispatched at the agent level
(needs the GUI callback), mirroring the old read_preview special path."""
action = (args.get("action") or "").strip()
if action == "open":
return preview_open(url=args.get("url", ""), label=args.get("label", ""))
if action == "close":
return preview_close(url=args.get("url", ""))
if action == "read":
return tool_error(
"preview read must run inside a desktop session (no GUI callback here)."
)
return tool_error("action must be one of: open, close, read.")
PREVIEW_SCHEMA = {
"name": "desktop_preview",
"description": (
"The preview pane beside the chat in the Hermes desktop app. open: show "
"a web URL (bare domains fine), a localhost dev server, or a file path "
"(HTML renders live) — opens for the current window only. close: dismiss "
"the whole pane, or one tab via url. read: what the pane currently shows "
"— returns {kind, url, title, text, start, end, total_chars}; a Browser "
"tab's text is the rendered page's visible text, paged with start/count "
"(char offsets); a file tab answers identity only (read the file with "
"read_file)."
),
"parameters": {
"type": "object",
"properties": {
"action": {"type": "string", "enum": ["open", "close", "read"]},
"url": {
"type": "string",
"description": "open: the target. close: one tab (omit for the whole pane).",
},
"label": {"type": "string", "description": "open: optional tab label."},
"start": {"type": "integer", "description": "read: 0-indexed char offset."},
"count": {"type": "integer", "description": "read: chars to return (capped per read)."},
},
"required": ["action"],
},
}
registry.register(
name="desktop_preview",
toolset="desktop_ui",
schema=PREVIEW_SCHEMA,
handler=_handle_preview,
emoji="🖼️",
)
+27 -43
View File
@@ -139,59 +139,43 @@ def project_switch(project: str, task_id: Optional[str] = None) -> str:
return json.dumps({"success": True, "id": proj.id, "slug": proj.slug, "name": proj.name, "primary_path": primary})
registry.register(
name="project_list",
toolset="project",
schema={
"name": "project_list",
"description": "List the desktop Projects (named workspaces) and which one is active.",
"parameters": {"type": "object", "properties": {}},
},
handler=lambda args, **kw: project_list(task_id=kw.get("task_id")),
)
def _handle_project(args, **kw):
action = (args.get("action") or "").strip()
tid = kw.get("task_id")
if action == "list":
return project_list(task_id=tid)
if action == "create":
return project_create(name=args.get("name", ""), path=args.get("path"), task_id=tid)
if action == "switch":
return project_switch(project=args.get("name", ""), task_id=tid)
return json.dumps({"success": False, "error": "action must be one of: create, switch, list."})
# Consolidated (#95681, maintainer-directed): project_list/create/switch each
# re-taught "desktop Projects (named workspaces)"; one action enum says it
# once (244 -> ~145 tok).
registry.register(
name="project_create",
name="desktop_project",
toolset="project",
schema={
"name": "project_create",
"name": "desktop_project",
"description": (
"Create a desktop Project (a named workspace) and switch this chat into it. "
"Pass `path` to anchor it to a repo/folder — this chat's workspace moves there "
"and the sidebar follows. Use when starting work in a new repo/folder; this is "
"the intentional way to move the session, not `cd`."
"Desktop Projects (named workspaces). create: make one and switch "
"this chat into it — pass path to anchor it to a repo/folder (the "
"chat's workspace moves there, the sidebar follows). switch: move "
"this chat into an existing project by name/slug/id — the "
"intentional way to move the session, not `cd`. list: all "
"projects + which is active."
),
"parameters": {
"type": "object",
"properties": {
"name": {"type": "string", "description": "Human name, e.g. 'Aurora Demo'"},
"path": {"type": "string", "description": "Primary repo/folder to anchor the project to"},
"action": {"type": "string", "enum": ["create", "switch", "list"]},
"name": {"type": "string", "description": "create: human name. switch: name, slug, or id."},
"path": {"type": "string", "description": "create: repo/folder to anchor to."},
},
"required": ["name"],
"required": ["action"],
},
},
handler=lambda args, **kw: project_create(
name=args.get("name", ""), path=args.get("path"), task_id=kw.get("task_id")
),
)
registry.register(
name="project_switch",
toolset="project",
schema={
"name": "project_switch",
"description": (
"Switch this chat into an existing desktop Project (by name, slug, or id). "
"Moves the session's workspace to the project's primary folder and the sidebar "
"follows. The intentional way to move between projects, not `cd`."
),
"parameters": {
"type": "object",
"properties": {
"project": {"type": "string", "description": "Project name, slug, or id"},
},
"required": ["project"],
},
},
handler=lambda args, **kw: project_switch(project=args.get("project", ""), task_id=kw.get("task_id")),
handler=_handle_project,
)
+2 -11
View File
@@ -81,14 +81,5 @@ READ_PREVIEW_SCHEMA = {
}
registry.register(
name="read_preview",
toolset="desktop_ui",
schema=READ_PREVIEW_SCHEMA,
handler=lambda args, **kw: read_preview_tool(
start=args.get("start"),
count=args.get("count"),
callback=kw.get("callback"),
),
emoji="🔍",
)
# Registration removed: consolidated into the `preview` tool (#95681);
# this module keeps its functions for the agent-level preview action=read dispatch.
+4 -6
View File
@@ -53,12 +53,10 @@ def read_terminal_tool(
READ_TERMINAL_SCHEMA = {
"name": "read_terminal",
"description": (
"Read what's currently shown in the in-app terminal pane of the Hermes "
"desktop GUI (the embedded shell beside this chat). Call with no arguments "
"to get the visible screen plus the total line count (`total_lines`). To "
"page through scrollback, pass `start_line` (0 = oldest line) and `count`; "
"valid lines are [0, total_lines). Returns JSON: "
"{total_lines, start, end, viewport_rows, cursor_row, text}."
"Read the in-app terminal pane beside this chat. No args = visible "
"screen + total_lines; page scrollback with start_line (0 = oldest) "
"+ count. JSON: {total_lines, start, end, viewport_rows, cursor_row, "
"text}."
),
"parameters": {
"type": "object",
+6 -11
View File
@@ -43,17 +43,12 @@ def read_window_below_tool(callback: Optional[Callable] = None) -> str:
READ_WINDOW_BELOW_SCHEMA = {
"name": "read_window_below",
"description": (
"Identify the application window directly underneath (behind) the "
"Hermes desktop window — what the user is working in behind this app. "
"Returns JSON: {window: {app, title, bounds{x,y,width,height}, id}, "
"frontmost: {app, title}, platform}. `title` may be empty when the OS "
"withholds window titles (e.g. macOS without the Screen Recording "
"permission — never prompted for, noted in `note`). Other Hermes "
"windows are skipped: the nearest non-Hermes window is reported. "
"Returns {error, platform} instead where the OS cannot enumerate "
"windows at all (e.g. a Wayland session); `error` says what would fix "
"it, so relay it rather than retrying. "
"Metadata only; this never captures pixels or content of other windows."
"Identify the app window directly behind the Hermes desktop window "
"(what the user is working in). JSON: {window: {app, title, bounds, "
"id}, frontmost, platform}. title may be empty when the OS withholds "
"it (noted in `note`); where windows cannot be enumerated at all, "
"{error, platform} says what would fix it — relay that instead of "
"retrying. Metadata only; never captures pixels."
),
"parameters": {
"type": "object",
+9 -26
View File
@@ -74,45 +74,28 @@ def setup_mcp_tool(
SETUP_MCP_SCHEMA = {
"name": "setup_mcp",
"description": (
"Propose an MCP server to the user as an inline consent card in the "
"Hermes desktop chat. The card lets them install a catalog entry, "
"re-enable a disabled server, or run an OAuth login — right there, "
"without opening the Capabilities tab — and blocks until they act or "
"decline. Use when the user asks to add/set up an MCP (e.g. \"add the "
"linear mcp\"), or when a task clearly needs one that is missing or "
"unauthorized. Never call it twice for the same server after a "
"decline. Returns JSON {status: installed|enabled|authorized|declined|"
"unanswered|error, server, detail?, tools?}. On declined/unanswered, "
"continue without the server. Catalog names: run `hermes mcp catalog` "
"in the terminal to list them."
"Propose an MCP server as an inline consent card (install a catalog "
"entry, re-enable a disabled server, or run OAuth); blocks until the "
"user acts. Use when they ask to add an MCP or a task clearly needs "
"a missing one. Never re-ask after a decline — on declined/"
"unanswered, continue without it. Catalog names: `hermes mcp "
"catalog` in the terminal."
),
"parameters": {
"type": "object",
"properties": {
"server": {
"type": "string",
"description": (
"The server's catalog name (for install) or its name in "
"mcp_servers config (for enable/authorize)."
),
"description": "Catalog name (install) or mcp_servers config name (enable/authorize).",
},
"action": {
"type": "string",
"enum": ["install", "enable", "authorize"],
"description": (
"install: add a catalog entry (prompts for any required "
"keys). enable: re-enable a disabled configured server. "
"authorize: run the OAuth browser flow for a configured "
"server. Defaults to install."
),
"description": "Defaults to install.",
},
"reason": {
"type": "string",
"description": (
"One short sentence shown on the card: why this server "
"helps right now (e.g. \"To read the JIRA ticket you "
"linked\")."
),
"description": "One sentence on the card: why this helps right now.",
},
},
"required": ["server"],
+10 -16
View File
@@ -62,38 +62,32 @@ def tip_tool(text: str, selector: str, title: str = "", side: str = "") -> str:
TIP_SCHEMA = {
"name": "tip",
"description": (
"Point at one thing in the Hermes desktop UI with a small accent-lit "
"bubble and an arrow — no dimming, no spotlight, no Next/Prev. Reach "
"for it when a sentence would be clearer with a finger on the thing "
"it's about: 'the model name is a button', 'your files are in here'. "
"Call tour(action='targets') first to see what's on screen and prefer "
"a target reporting `stable: true`; never guess a selector. One tip at "
"a time — a new one replaces the last. Say the same thing in chat as "
"well; the bubble is a pointer, not the message. Use it sparingly: a "
"bubble on every turn is what makes people stop reading them."
"Point at one thing in the desktop UI with a small arrow bubble (no "
"dimming, no tour chrome) — for when a sentence is clearer with a "
"finger on its subject. Get selectors from tour(action='targets'), "
"prefer stable:true, never guess. One tip at a time (new replaces "
"last); say the same thing in chat too — the bubble is a pointer, "
"not the message. Sparingly: a bubble every turn stops being read."
),
"parameters": {
"type": "object",
"properties": {
"text": {
"type": "string",
"description": "The one line the bubble says. Keep it to a sentence.",
"description": "The one-sentence bubble text.",
},
"selector": {
"type": "string",
"description": (
"CSS selector of the element the arrow points at, from "
"tour(action='targets')."
),
"description": "Selector from tour targets.",
},
"title": {
"type": "string",
"description": "Optional short heading above the text.",
"description": "Optional heading.",
},
"side": {
"type": "string",
"enum": list(SIDES),
"description": "Preferred side of the element. Omit for 'top'; it flips at a screen edge either way.",
"description": "Omit for 'top'; flips at screen edges.",
},
},
"required": ["text", "selector"],
+23 -27
View File
@@ -112,37 +112,33 @@ _STEP_SCHEMA = {
"properties": {
"selector": {
"type": "string",
"description": "CSS selector of the element this step highlights. Omit for a centered narration-only step.",
"description": "Element to highlight; omit = centered narration.",
},
"title": {"type": "string", "description": "Popover title."},
"text": {"type": "string", "description": "Popover body text."},
"text": {"type": "string", "description": "Popover body."},
"side": {
"type": "string",
"enum": list(SIDES),
"description": "Preferred popover side. Omit to auto-place.",
"description": "Popover side; omit to auto-place.",
},
},
}
TOUR_SCHEMA = {
"name": "tour",
# Dieted (#95681): targets-first flow + stable-selector preference kept
# (pre-effect: skipping them means guessed selectors on re-rendering UI).
"description": (
"Give a live guided tour in the Hermes desktop GUI: dim the screen, "
"highlight an element, and attach a popover with your own title/text. "
"Works on two surfaces — 'app' (the Hermes app itself) and 'preview' "
"(whatever page is open in the in-app browser, so any web app can be "
"toured). ALWAYS call action='targets' first to discover what is on "
"screen instead of guessing selectors; each target reports "
"`stable: true` when its selector keys off identity (data-tour, id, "
"data-testid, aria-label) and survives a re-render — prefer those, and "
"re-scan if a selector stops matching. Then either narrate at your own "
"pace with action='show' (one highlight per call — replaces the "
"previous one; pair each with a chat message describing it), or hand "
"control to the user with action='start' + a steps array (driver.js "
"renders Next/Prev buttons; 'next'/'prev' also page it "
"programmatically). action='stop' clears the tour. Use when the user "
"asks how something works, where something is, or for a walkthrough of "
"an app or workflow."
"Guided tour in the desktop GUI: dim the screen, highlight an "
"element, attach a titled popover. Surfaces: 'app' (Hermes itself) "
"or 'preview' (the page in the preview pane). ALWAYS call "
"action='targets' first — prefer targets marked stable:true (their "
"selectors survive re-renders); re-scan if one stops matching. Then "
"narrate with action='show' (one highlight per call, replaces the "
"last — pair each with a chat message) or hand over with "
"action='start' + steps (user gets Next/Prev; 'next'/'prev' also "
"page it). 'stop' clears. Use for how-does-X-work / where-is-Y "
"walkthroughs."
),
"parameters": {
"type": "object",
@@ -150,32 +146,32 @@ TOUR_SCHEMA = {
"action": {
"type": "string",
"enum": list(ACTIONS),
"description": "targets: list tourable elements. show: highlight one element. start: begin a multi-step user-paced tour. next/prev: page a started tour. stop: end the tour.",
"description": "targets first; show narrates; start hands over.",
},
"surface": {
"type": "string",
"enum": list(SURFACES),
"description": "Where the tour runs: 'app' (Hermes desktop UI, default) or 'preview' (the page in the in-app browser pane).",
"description": "'app' (default) or 'preview'.",
},
"selector": {
"type": "string",
"description": "For show: CSS selector of the element to highlight (from action='targets', preferring a stable one). Omit for a centered narration popover.",
"description": "show: selector from targets (prefer stable). Omit = centered narration.",
},
"title": {"type": "string", "description": "For show: popover title."},
"text": {"type": "string", "description": "For show: popover body text."},
"title": {"type": "string", "description": "show: popover title."},
"text": {"type": "string", "description": "show: popover body."},
"side": {
"type": "string",
"enum": list(SIDES),
"description": "For show: preferred popover side. Omit to auto-place.",
"description": "show: popover side; omit to auto-place.",
},
"steps": {
"type": "array",
"items": _STEP_SCHEMA,
"description": "For start: the ordered tour steps.",
"description": "start: ordered steps.",
},
"step_index": {
"type": "integer",
"description": "For start: 0-indexed step to begin at (default 0).",
"description": "start: 0-indexed first step.",
},
},
"required": ["action"],
+2 -2
View File
@@ -236,7 +236,7 @@ TOOLSETS = {
"project": {
"description": "Desktop Projects — create/switch named workspaces (GUI sessions only)",
"tools": ["project_list", "project_create", "project_switch"],
"tools": ["desktop_project"],
"includes": []
},
@@ -253,7 +253,7 @@ TOOLSETS = {
"description": "Desktop GUI affordances — in-app terminal/browser panes, pane focus, reactions (GUI sessions only)",
"tools": [
"read_terminal", "close_terminal",
"open_preview", "close_preview", "read_preview", "drive_preview", "annotate_preview",
"desktop_preview", "drive_preview", "annotate_preview",
"read_window_below",
"focus_pane", "react_to_message",
"setup_mcp", "tour", "tip",